{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "### 数据处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 安装类库\n",
    "# !mkdir /home/aistudio/external-libraries\n",
    "# !pip install imgaug -t /home/aistudio/external-libraries\n",
    "import sys\n",
    "sys.path.append('/home/aistudio/external-libraries')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "image shape: (32, 32, 3)\n",
      "label value: horse\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAMgAAADFCAYAAAARxr1AAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAHC1JREFUeJztnXuMXHd1x79nZu689732erO24zixnZg8XGEgFNTybAOqFKhaBH9U+SMCKoFUBP9EVGqp1EpUKqD+UVEFNSKVKIEWUFIaKCGiStNAHiTEeZgkdmJnba937X3NzO687+kfMw47+z17Pdldr3fN+UjW7h7fufd378yZe89bVBWO49jELvcCHGcz4wriOBG4gjhOBK4gjhOBK4jjROAK4jgRuII4TgSuII4TwZoURERuE5GXROSYiNy1XotynM2CrDaSLiJxAC8D+CCAUwCeBPAJVX1xpdfEg4QGqaBDlslkaLtYTEhWqVRIptowjxOG/Pp4LE6yIJkgWT7XY+2RJLXFKskaTd5OY/b1jRtfTf2D20iWyeR4NSHvs1xeNGQLJEun+XpjhY9Ao9kkWRAEJEum0vYO6DB8IDWurSrLwpBlANA0rnmjXiNZPN75/k9PnkepUOQPyjL4E9I9bwdwTFVfBQARuQ/A7QBWVJAgFWDnjXs7ZDfd9BbaLpNJkuyV40dJVq/NmseplPj1OeODv2v3MMnecevvkUxCvuCnnjlOsvOzJZI1svxaAMjz5x5//PE/J9lbbnonycqLvM8jzz9FsueeY9nBA3y9LYUDgNm5Asm2je4k2dV795FMhL8B6k3+UqmDv/iqTVbshcWyucbSLMunz46TrL833/H3333+S+b+lrOWR6wxAEtXcqot60BEPiUiT4nIU806fyM5zmbmkhvpqnq3qh5W1cPxgB9zHGczs5ZHrNMAdi35e2dbtiIahqhXOm+JszPnaLvkju0kGx1h2bmpunmcuXN8i85m2V5ZWJwh2anTL5NsbIRujEinUyRTnSfZ4NCAuca3HtrPxxkdJVnTeJ6ePc/X7OWjL5FsboYfkRYW+HGqt99eY98wPwdWQ76O52fPkyyMZ0kmYtkLbDtVq/yoWq7aj6qLNd5nkOZ1yzL756LGR5u13EGeBLBPRK4RkSSAjwN4YA37c5xNx6rvIKraEJHPAvhvAHEA96jqC+u2MsfZBKzlEQuq+iCAB9dpLY6z6fBIuuNEsKY7yJslnohjaLivQ1avsh87YQTXioU5klWMeAAADA8MsWw4T7KbbmGfPowg5dnJUyTLBmykZ7MchGs07DWm0xyrSSX42DHl2EEixobywQPXkWz3VVeRLNfDwcggawVHgcoiOx1qjSLJ5ubZ2TGzwO+X5XAIhM9lscD7q4Z2iKBoxA8b81Mkyy6Lt1SNwLOF30EcJwJXEMeJwBXEcSJwBXGcCDbUSE+nUrju2j0dsiDJ6SepFMvm56dJ1qzb2bxqJBcOD/WT7PoDe0k2PcsJkM8+8wqvMdhBsr7+XpIVY2ysAnZUOQZedyrBstERjhTnMntI9sJznBXQk+HM2zDJDgcAiNVYnkuwsZxK8vdsocTnVyxwhoOEbCxXipwBsNiwsyZmjIzjsMjv4WK9MzrfWOGzsxy/gzhOBK4gjhOBK4jjROAK4jgRbKyRnk7hwP5rO2S5HEef6w023IoFjo42a3Z0dWqCX79jhKsHBw2jenqanQGJBDsNjII5hIYhGQ/s76Aw5G1j4KyCVJIj6VYFYCPNsljIr80aRvZ8jVPOAaA4z8ZyT9pwLoR8jqkGy3LCH7eZeY7MWxF3GE4NAIgpX8dymdPlK4VOw90qJzb339VWjvNbiiuI40TgCuI4EbiCOE4EriCOE8GavFgicgJAEUATQENVD19ke6SWNWtLpXgJVg+rgzccIFk6sJd/8jinGoyNctOHIMGvtxrMZY30jHiTv1uGhvpIFvbY3pJUiutBmk32vtSqRhO8uLFuw0OUCtjzk0/yeqbO83EB4PzZsyyrsmercJZ7dZQWjfUY1/HE+DGS7drDdSw92zlVCADiFfZ4FYxGIPW5ztqWZqO7VJP1cPO+V1W5rYXjXAH4I5bjRLBWBVEAPxGRX4rIp6wNlnZWXCjZASnH2ays9RHr3ap6WkS2A3hIRH6tqo8s3UBV7wZwNwCM7R71mdPOlmKtbX9Ot39OicgP0Gpo/chK28cTCfQNdqZ81BucXlGscPrBVbvYcOtL2c0GqsUTJBscYCMvFuMbaCbLxvPAIDd86AlZtnPn1SQLM7YxmO8zajCMdn+lBW6cEDcaSzRqnLJjpchUjQ6FzzzxS3ONP3/61ySrL/B7M33mdX5xjGtWtg9xus/4aTbSc3l+X7btto30nGFsx4x6oHBZR8hupxqs+hFLRHIi0nPhdwB/AOD51e7PcTYja7mDjAD4gYhc2M+/qeqP12VVjrNJWEvr0VcB3LKOa3GcTYe7eR0ngg2tB4kl4sgNDXbIzkycoO3OzXPccWgHt+iPiz36K2M0Ieg3WvxncmyQDxl1I8UFrqvIGEb60DAfI8jbl1gDNi6TaXY61OtsFFtdAWtlNtwLJa6VeOKJX5DsJw/afpXZOTZ2G4aRX2uywRsPeLuRPv4+Tjf5PTh/hiP4e27gJhkA0GvU2+SM0XoVGvV2iY10x/ltwBXEcSJwBXGcCFxBHCeCDTXSIYBkOtOtYxk2LhNGNHtukY1VNYxDAIgbowV6BzgSn+/laO9rpzhV/twkp4hnhQ3gA9ewkT02yvMNAWCxyeeTSPB6YtY45TpHyE+f4pEBD/2EDfJnn+Xo+My0nSOXTnH6ftMYV2AFpStlw7mwwGn1mRjPXS+e59eePn7GXOPYGI9zyKZ4PuJcrHOf1phqC7+DOE4EriCOE4EriONE4AriOBFsbCQ9rgjyncbtjt2DtF3/do5Sq5XWXLaN9FyMU6Mnpzmye+Qod2t87DFOSD53miP7KWXjcnGGL+d7/9AeLbDnwC6SBXHeJ2KcLTB+jlPgf/xfPyfZIz/jNPZa0xgjYJwLANTrvG29wVkFVaM8oVbj2vXZWXYu5AM+v1jduLbG+wcAsxk+djLJRnoq0/mZEqPUwcLvII4TgSuI40TgCuI4EbiCOE4EFzXSReQeAH8EYEpVb2zLBgF8B8AeACcAfExVOQS9jFAbqDQ6N0ulOXocpA0jK8HGbio0irgBLM5wJP34S9xM7MXnx0k2foKN0ECHSDY7x4byo2d/xdsVuOYeAMb2cvp2b5bPJybsiDh69FWSPf5/L5Bs0QiQJ1NsFDdWyEioGvMDq1WeM1iu8BzGWIJfO7c4SbIwxSMotvezAyMd8GcCAAaG2MlTNcZQ9IWd6e4Jo2mgRTd3kG8CuG2Z7C4AD6vqPgAPt/92nCuOiypIu43P8kSf2wHc2/79XgAfWed1Oc6mYLU2yIiqTrR/P4tWAweTpY3jinN8e3aczcyajXRtNRhasX5RVe9W1cOqerinn+0Nx9nMrDaSPikio6o6ISKjADgkbdBo1HFuqrMT+LZhTkNv1NlYLRr914KQI7MAcPQZnjOICjd1G+jlVPT5XjbSk0Y0e7ZiNG9juxTT5+xU8mdffJRkxblTJDO/wYzIdyLOqfZ9vbzuUDl1X41GawBQq7ODoW7Jmvxk8NZ33UiyXaOjJHv28SMkKzU5Cl+Hvcbd+/k9jBkTA0bLne/rI//5U3N/tK+utmIeAHBH+/c7ANy/yv04zqbmogoiIt8G8HMAB0TklIjcCeDLAD4oIq8A+ED7b8e54rjoI5aqfmKF/3r/Oq/FcTYdHkl3nAg2NN292WiiNNtpgMUavIRGg0eHzc5w5Lo4YxtuR37BUfMbr+URbHWjIdz0JDct6+/lZnK5PHdTLysb5PkcR3oBIDHN9dnlEn9fNRt8jrkcHzubZsM9k+GMgmKRzzkmfL0BIJ1mI79mOFD6hrk84a3vexvJ9h+4hmQDu7jZ3q+feY1khYadqCF5jppXxYjilzqj/U3tbgSb30EcJwJXEMeJwBXEcSJwBXGcCFxBHCeCjfVi1ZuYmez0JhSmOU2hYTRomJpkz1S5YKeaTIyzx6t6/imShVVOISsWjayZkNMzrtrOnq258+zFajT4tQDQ28OvX+znGpGFItdaiPG2NZrs7Yon2OMUGKMhEgl7jERvH3vB4lPcJGH3wb0ku/7wQZJl8vx+Hfr9m0m2fYzTj8oLdqJrWYzmEMZcx3OLndexERrNKwz8DuI4EbiCOE4EriCOE4EriONEsKFGepAMMDq2s0MWj/MSyiVOw0iBDbyzdTbQAKBhGHQnTj7N60kY8+16uYlAIsWGdt0YX1AqcR1KKrXfXOPeEa5PqRsNERp1TgOxjO+E0RwxNL7+8oOcFjKQ5RECAJBO8XGu2suG++9+6ADJ9ozy6ISGsmGsed5fj5XGU7WbX4QBf1Z6E7zPhnTuM2FcQwu/gzhOBK4gjhOBK4jjROAK4jgRrLaz4pcAfBLAhfD2F1X1wYvtK5VO49r9+zr3H2PjOy28rJRRsjD+GnfqA4AnHzLqCco8w8/q+t8/xMZlT54N26mJCZLVa5wB0GzatRbWPnft3E0y6xtMwcauGDUdxQWOwvf0sAE8Mmwb6YFxfXbdzI0Xxq7jcRNiRPaTcX6v40k+iHXNYkn7OlaMcQ75OGdIxJcdO2E4hyxW21kRAL6mqofa/y6qHI6zFVltZ0XH+a1gLTbIZ0XkiIjcIyJcN9lmaWfFwqx3VnS2FqtVkK8DuBbAIQATAL6y0oZLOyv2DnhnRWdrsapIuqq+YR2LyDcA/LCb1zWbTRSLnZHPhGGklSps4KWNsPDcrB1Jr1aM2Xp1Y8ahFUnPGI0KFrgJwNkz3NxBjZl+J4+fNNdYNjoz5rOcij62g9PiFRzZr1Q51f7ka6+QLJviYyyk7Sfo/DB/oSX7uIPjbJmPrWU2qhPGDMbAMNyThndAQnuOYgzGWITQyD6QZe9/d4H01d1B2u1GL/BRADz50nGuALpx834bwHsADIvIKQB/DeA9InIIrabVJwB8+hKu0XEuG6vtrPgvl2AtjrPp8Ei640SwoenuoSrK1U6DN6yyAZxPGnP0jLrwhUU7BbpaZWO5abzeqhefL7DBGoYcmZ2b5/mGFcNYDWPcyRAAEkZkuGGk2sfFqivnt21+jjsPnp/kaH82zt+J9Yp9Hfvi7L3fBc4ACMt8fZqGoZywMgCMqHnGSFdvjaFhmgHLJcay2LKMDVFPd3ecNeMK4jgRuII4TgSuII4TwYYa6QAgyyKnWaPFfjpgIy2jrMuJuDGLEECzueJM0Q7qxmiB6WmOkI/tYsP0lrdxczOJs+F3Zpwb3gHAyVNPkiyTYaO4boyCSKf4mtWNKH61xpkGxTmOpDcCY7gigL4UR6nDODs2qg2+3g3DSA/rfL1jhpFeEz6X8qK9Rg34mieTnDWRS3duZzleLPwO4jgRuII4TgSuII4TgSuI40SwoUa6GpH0uhqpyYb9FMTZcIdRzw4AYjSZswKxahiSuTwbwB/90w+QbO8NbKTHU7zGI0+9YK7xpz96lGTnjbmFzTobxc2YUfsOI3KdYdlChQ337UOcUg8AB27m5na9/Wy4F6rcRM+aAVirsaFdN0obrBT4utHxHwDCqpXaztkL1WWN4pordN1fjt9BHCcCVxDHicAVxHEicAVxnAi6qSjcBeBfAYygVUF4t6r+o4gMAvgOgD1oVRV+TFXtae9tFECITmu50WTjS40UbyvwWavahlutyoafsUuI8fWwbz+PE9t/kLuXZ4b50tWUDb/D736Hucb9+99CsvEzZ0h25ixH9qF8bA35BH90/0Mkm3iZR8wNjnCaPQBs38kN4RZKPN4sbPJ5xwI2nkWMN9H4BBaMLvdN4xgAkBI26GOGR6Za7vysrGckvQHgC6p6EMCtAD4jIgcB3AXgYVXdB+Dh9t+Oc0XRTeO4CVV9uv17EcBRAGMAbgdwb3uzewF85FIt0nEuF2/KBhGRPQB+B8DjAEZU9ULJ2lm0HsGs17zROG5hnivuHGcz07WCiEgewPcAfE5VO6JN2qqHNB/qljaOy/UZPYwcZxPTlYKISICWcnxLVb/fFk9e6I/V/mkMGHecrU03XixBq83PUVX96pL/egDAHQC+3P55fxf7QrC8a55RQ9E00k+MjAvMTdu9fms1ozmA0azAcm3tNobYZ1PcYXBhntNCakZjiOQKLfwG8lz70bOfuxaODA2SzPLAGGUjeOx/HiPZuOE17OmxuxbmjIYIzRKncRiTF8y0m5rR/TFjdVFM8zWrGesGgIRx4mJ8fpZ7T1d44OH9d7HNuwD8GYDnRORXbdkX0VKM74rInQBOAvhYV0d0nC1EN43jHsXKnUzfv77LcZzNhUfSHScCVxDHiWBjmzaoUlqC1cp/ocx6qyHXaVTmbUMrtOYCxvgpMWs0Jbj6Kp7BF6vwGmWBj50MOe0hZRW3AAgCdiRk49wcIhXjc6k0uKaj1GDjOWk4NpJxbtpwwzVj5hr3DhqpJkleY8nocDlvzGusFK2RCMZ7ZaSkiJUXBHPSAerG3MJmo3ON4QqdGpfjdxDHicAVxHEicAVxnAhcQRwngg1v2tCsdBqTGhhjCWrGDL4iyyZP210L1Yg0G+US6O/rI9mQIZs7a3RwDPnSpRNs9DcadkfAWNwwqo13I6izAVwpcwaBGk0S1Oh4GBhNKQa28zkDQDrNC4oLG/mG/wN9yhHyoQbLJie43qVhOBwq4QpNGwzj3TL8q6Vl74OVemDgdxDHicAVxHEicAVxnAhcQRwngg010mMKpJbZ2rGAo89loxnDzAQb5Oen7BIUK7PS6heQy7JRvVBkA/i1l06QLBnj1w71cYfChDEvDwDCpJFKXj1PMjWM/KqVX57hlPy48NubyhkNH8RuiLBY4JR+VTbSYaXQxzMkS6d5jWEPOwikOMfHLdtGdWG58Q0gnTQyMYqdn7NY02cUOs6acQVxnAhcQRwnAlcQx4lgLZ0VvwTgkwAuWM9fVNUHo/YVgyAbdkZTa2WOmqLE6cqL57ijX21hhSi1YaaLUX8exI3uiGXepxjp94UK11c3Cmzs9vWyYQoAkjVq8RdneJ81Pk49yfXe8STXs4sR7e/NGPMf67YBXJ/itHoFG+lqpOTPhvwe1ow5ijEjWp8O2cCPG+cMAIkGX8d4jR0g8WrnumPanZHejRfrQmfFp0WkB8AvReRCT8uvqeo/dHUkx9mCdFOTPgFgov17UUQudFZ0nCuetXRWBIDPisgREblHRLiPDTo7K5YK3lnR2VqspbPi1wFcC+AQWneYr1ivW9pZMd/rnRWdrUVXkXSrs6KqTi75/28A+OFFd9QEYsuCs0GCI+m94LrnoMLR1WqJU8EBIB5jvQ+SbFymAjb80sZ2PSlOES812blQmOdZfcV5eyJEIsavv26Uo8qZPF+LQoGj/fNnOKtgbpaN7IEcG8ADwjIAyC4YWQ7GTMHQmIVYXp4yAaBujJRU4/2PG3XziRWy061ov3FpkdTO91pW7GS1bF8X22ClzooX2o62+SiA57s6ouNsIdbSWfETInIILdfvCQCfviQrdJzLyFo6K0bGPBznSsAj6Y4TwcamuyOGnHYanaEx1y+RZG/X7iFeal/uFfM4QYIN46TRRTxhDKxfWGCjr1nj6HqQYMO2Z4Cj2aFRXw0AiwuGgyHPRnoixw6CsMIW6+lxNtKnZ9mxMTy8m2SBER0HgGST5ZUqn0+xzBkAlV6+tknj+sTT/L7US+wIaNTt62hlJKiRYLE84N5d2zi/gzhOJK4gjhOBK4jjROAK4jgRbLiRntJOAzxmdD/XGhtuuYANvH3X7jOP0zQisePjp0lWKnDk+9grx3h/Rl14Is7G8+g2zuHsMYxsANi2nUer1ZJs+JeEjV1N83YNI50/leXtakaWQV3sVHKNG6n6Rnc7aXBkv17kKH4s4NcGxrp7jcyFecNRAgDxPl57o2o0I5xflmpvHNfC7yCOE4EriONE4AriOBG4gjhOBK4gjhPBxs4oBKDxi8f800b6wZ69O0nWv2ObeYx3LrLH49H/fYxkrx1nj1WtZgyhr3KzgWrIqQ+v13i7dMZO45DcdSTLp4zzMboEJvvZO9U7xJ6fwSE+9sIie5ymZ20P0eCeHSQLA15PptpLsqDGXqJKhWVVNeYJZvlDsWCMdwCAsuGMSuc5VWm5E1SsmQ0GfgdxnAhcQRwnAlcQx4mgm5LbtIg8ISLPisgLIvI3bfk1IvK4iBwTke+IrBCOdZwtTDdGehXA+1S11G7e8KiI/AjA59FqHHefiPwzgDvR6nSyIqqKeqPT2LLqNNIpNi6DGG+X7eGGBgCwQ1nvt/fdRrLx1zn9pDfPKS1Tr79Gsvl5rrUI8mw8V0N7tEDd8E5UjHmEGuO3qFTiFJmEkbJz4/XXkKxvgJ0dg/08tgEApit8nB1Xs+FeOM0Oi8lZHuVQqLOhXTMcMlpnAzrM2N/lqQR/VqZnuT7l9MlTHX9XalxzYnHRO4i2uFBFFLT/KYD3AfiPtvxeAB/p6oiOs4XoygYRkXi7YcMUgIcAHAcwp78ZrXoKK3RbXNo4rlBiF6PjbGa6UhBVbarqIQA7AbwdwPXdHmBp47jevN3I2XE2K2/Ki6WqcwB+BuCdAPpF3pjxtRMAP9A7zhanm/EH2wDUVXVORDIAPgjg79FSlD8BcB+AOwDcf9GjqSBe7zxkLMFLiBk1CzWjaD9pl1pAjOHyO4c4Sj02OEKywjyPWehL8P5mZqdJVrIaCxj1EwAQGl9NWmFDu2p0MkwbcwL7R7gZw/49fKOvG0a/Vm2DNd/LDot8H1/06gzXrFTBEfIzU5MkKxmG++AY18oEeTbmASAEX/OUUQ/UM9DZOjpuNOyw6MaLNQrgXhGJo3XH+a6q/lBEXgRwn4j8LYBn0Oq+6DhXFN00jjuCVkf35fJX0bJHHOeKxSPpjhOBK4jjRCCq3faYW4eDiZwDcBLAMAAOtW5N/Fw2Jxc7l6tV1a6XWMKGKsgbBxV5SlUPb/iBLwF+LpuT9ToXf8RynAhcQRwngsulIHdfpuNeCvxcNifrci6XxQZxnK2CP2I5TgSuII4TwYYriIjcJiIvtUt179ro468FEblHRKZE5PklskEReUhEXmn/HIjax2ZBRHaJyM9E5MV2KfVftOVb7nwuZVn4hipIO+HxnwB8CMBBtCblHtzINayRbwJYXrt7F4CHVXUfgIfbf28FGgC+oKoHAdwK4DPt92Irns+FsvBbABwCcJuI3IpW1vnXVPU6ALNolYW/KTb6DvJ2AMdU9VVVraGVKn/7Bq9h1ajqIwCWFzzfjlbJMbCFSo9VdUJVn27/XgRwFK2q0C13PpeyLHyjFWQMwPiSv1cs1d1CjKjqRPv3swC4yGSTIyJ70MrYfhxb9HzWUhYehRvp64i2fOZbym8uInkA3wPwOVXtmHqzlc5nLWXhUWy0gpwGsGvJ31dCqe6kiIwCQPsnz2PepLTbOH0PwLdU9ftt8ZY9H2D9y8I3WkGeBLCv7V1IAvg4gAc2eA3rzQNolRwD3ZYebwJERNCqAj2qql9d8l9b7nxEZJuI9Ld/v1AWfhS/KQsHVnsuqrqh/wB8GMDLaD0j/uVGH3+Na/82gAkAdbSeae8EMISWt+cVAD8FMHi519nlubwbrcenIwB+1f734a14PgBuRqvs+wiA5wH8VVu+F8ATAI4B+HcAqTe7b081cZwI3Eh3nAhcQRwnAlcQx4nAFcRxInAFcZwIXEEcJwJXEMeJ4P8BDOlRG0/6AGEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 216x216 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import paddle\n",
    "import numpy as np\n",
    "from PIL import Image\n",
    "import matplotlib.pyplot as plt\n",
    "import imgaug as ia\n",
    "import imgaug.augmenters as iaa\n",
    "\n",
    "# 读取数据\n",
    "reader = paddle.batch(\n",
    "    paddle.dataset.cifar.train10(),\n",
    "    batch_size=8) # 数据集读取器\n",
    "data = next(reader()) # 读取数据\n",
    "index = 4 # 批次索引\n",
    "\n",
    "# 读取图像\n",
    "image = np.array([x[0] for x in data]).astype(np.float32) # 读取图像数据，数据类型为float32\n",
    "image = image * 255 # 从[0,1]转换到[0,255]\n",
    "image = image[index].reshape((3, 32, 32)).transpose((1, 2, 0)).astype(np.uint8) # 数据格式从CHW转换为HWC，数据类型转换为uint8\n",
    "print('image shape:', image.shape)\n",
    "\n",
    "# 图像增强\n",
    "# sometimes = lambda aug: iaa.Sometimes(0.5, aug) # 随机进行图像增强\n",
    "# seq = iaa.Sequential([\n",
    "#     sometimes(iaa.CropAndPad(px=(-4, 4))),      # 随机裁剪填充像素\n",
    "#     iaa.Fliplr(0.5)])                           # 随机进行水平翻转\n",
    "# image = seq(image=image)\n",
    "\n",
    "# 读取标签\n",
    "label = np.array([x[1] for x in data]).astype(np.int64) # 读取标签数据，数据类型为int64\n",
    "vlist = [\"airplane\", \"automobile\", \"bird\", \"cat\", \"deer\", \"dog\", \"frog\", \"horse\", \"ship\", \"truck\"] # 标签名称列表\n",
    "print('label value:', vlist[label[index]])\n",
    "\n",
    "# 显示图像\n",
    "image = Image.fromarray(image)   # 转换图像格式\n",
    "image.save('./work/out/img.png') # 保存读取图像\n",
    "plt.figure(figsize=(3, 3))       # 设置显示大小\n",
    "plt.imshow(image)                # 设置显示图像\n",
    "plt.show()                       # 显示图像文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_data: image shape (128, 3, 32, 32), label shape:(128, 1)\n",
      "valid_data: image shape (128, 3, 32, 32), label shape:(128, 1)\n"
     ]
    }
   ],
   "source": [
    "import paddle\n",
    "import numpy as np\n",
    "import imgaug as ia\n",
    "import imgaug.augmenters as iaa\n",
    "\n",
    "# 训练数据增强\n",
    "def train_augment(images):\n",
    "    # 转换格式\n",
    "    images = images * 255 # 从[0,1]转换到[0,255]\n",
    "    images = images.transpose((0, 2, 3, 1)).astype(np.uint8) # 数据格式从BCHW转换为BHWC，数据类型转换为uint8\n",
    "    \n",
    "    # 增强图像\n",
    "    sometimes = lambda aug: iaa.Sometimes(0.5, aug) # 随机进行图像增强\n",
    "    seq = iaa.Sequential([\n",
    "        sometimes(iaa.CropAndPad(px=(-4, 4))),      # 随机裁剪填充像素\n",
    "        iaa.Fliplr(0.5)])                           # 随机进行水平翻转\n",
    "    images = seq(images=images)\n",
    "    \n",
    "    # 减去均值\n",
    "    mean = np.array([0.4914, 0.4822, 0.4465]).reshape((1, 1, 1, -1)) # cifar数据集通道平均值\n",
    "    stdv = np.array([0.2471, 0.2435, 0.2616]).reshape((1, 1, 1, -1)) # cifar数据集通道标准差\n",
    "    \n",
    "    images = (images/255.0 - mean) / stdv # 对图像进行归一化\n",
    "    images = images.transpose((0, 3, 1, 2)).astype(np.float32) # 数据格式从BHWC转换为BCHW，数据类型转换为float32\n",
    "    \n",
    "    return images\n",
    "\n",
    "# 验证数据增强\n",
    "def valid_augment(images):\n",
    "    # 转换格式\n",
    "    images = images * 255 # 从[0,1]转换到[0,255]\n",
    "    images = images.transpose((0, 2, 3, 1)).astype(np.uint8) # 数据格式从BCHW转换为BHWC，数据类型转换为uint8\n",
    "    \n",
    "    # 减去均值\n",
    "    mean = np.array([0.4914, 0.4822, 0.4465]).reshape((1, 1, 1, -1)) # cifar数据集通道平均值\n",
    "    stdv = np.array([0.2471, 0.2435, 0.2616]).reshape((1, 1, 1, -1)) # cifar数据集通道标准差\n",
    "    \n",
    "    images = (images/255.0 - mean) / stdv # 对图像进行归一化\n",
    "    images = images.transpose((0, 3, 1, 2)).astype(np.float32) # 数据格式从BHWC转换为BCHW，数据类型转换为float32\n",
    "    \n",
    "    return images\n",
    "\n",
    "# 读取训练数据\n",
    "train_reader = paddle.batch(\n",
    "    paddle.reader.shuffle(paddle.dataset.cifar.train10(), buf_size=50000),\n",
    "    batch_size=128) # 构造数据读取器\n",
    "train_data = next(train_reader()) # 读取训练数据\n",
    "\n",
    "train_image = np.array([x[0] for x in train_data]).reshape((-1, 3, 32, 32)).astype(np.float32) # 读取训练图像\n",
    "train_image = train_augment(train_image)                                                       # 训练图像增强\n",
    "train_label = np.array([x[1] for x in train_data]).reshape((-1, 1)).astype(np.int64)           # 读取训练标签\n",
    "print('train_data: image shape {}, label shape:{}'.format(train_image.shape, train_label.shape))\n",
    "\n",
    "# 读取验证数据\n",
    "valid_reader = paddle.batch(\n",
    "    paddle.dataset.cifar.test10(),\n",
    "    batch_size=128) # 构造数据读取器\n",
    "valid_data = next(valid_reader()) # 读取验证数据\n",
    "\n",
    "valid_image = np.array([x[0] for x in valid_data]).reshape((-1, 3, 32, 32)).astype(np.float32) # 读取验证图像\n",
    "valid_image = valid_augment(valid_image)                                                       # 验证图像增强\n",
    "valid_label = np.array([x[1] for x in valid_data]).reshape((-1, 1)).astype(np.int64)           # 读取验证标签\n",
    "print('valid_data: image shape {}, label shape:{}'.format(valid_image.shape, valid_label.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "### 模型设计"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import paddle.fluid as fluid\n",
    "from paddle.fluid.dygraph.nn import Conv2D, Pool2D, Linear, BatchNorm\n",
    "import math\n",
    "\n",
    "# 模组结构：输入维度，输出维度，滑动步长，基础长度\n",
    "group_arch = [(16, 16, 1, 18), (16, 32, 2, 18), (32, 64, 2, 18)]\n",
    "group_dim  = 64 # 模组输出维度\n",
    "class_dim  = 10 # 类别数量维度\n",
    "\n",
    "# 卷积单元\n",
    "class ConvUnit(fluid.dygraph.Layer):\n",
    "    def __init__(self, in_dim, out_dim, filter_size=3, stride=1, act=None):\n",
    "        \"\"\"\n",
    "        功能:\n",
    "            初始化卷积单元，H/W=(H/W+2*P-F)/S+1\n",
    "        输入:\n",
    "            in_dim      - 输入维度\n",
    "            out_dim     - 输出维度\n",
    "            filter_size - 卷积大小\n",
    "            stride      - 滑动步长\n",
    "            act         - 激活函数\n",
    "        输出:\n",
    "        \"\"\"\n",
    "        super(ConvUnit, self).__init__()\n",
    "        \n",
    "        # 添加卷积\n",
    "        self.conv = Conv2D(\n",
    "            num_channels=in_dim,\n",
    "            num_filters=out_dim,\n",
    "            filter_size=filter_size,\n",
    "            stride=stride,\n",
    "            padding=(filter_size-1)//2,                       # 输出特征图大小不变\n",
    "            param_attr=fluid.initializer.MSRA(uniform=False), # 使用MARA 初始权重\n",
    "            bias_attr=False,                                  # 卷积输出没有偏置项\n",
    "            act=None)\n",
    "        \n",
    "        # 添加正则\n",
    "        self.norm = BatchNorm(\n",
    "            num_channels=out_dim,\n",
    "            param_attr=fluid.initializer.Constant(1.0), # 使用常量初始化权重\n",
    "            bias_attr=fluid.initializer.Constant(0.0),  # 使用常量初始化偏置\n",
    "            act=act)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        功能:\n",
    "            对输入的特征进行卷积和正则\n",
    "        输入:\n",
    "            x - 输入特征\n",
    "        输出:\n",
    "            x - 输出特征\n",
    "        \"\"\"\n",
    "        # 进行卷积\n",
    "        x = self.conv(x)\n",
    "        \n",
    "        # 进行正则\n",
    "        x = self.norm(x)\n",
    "        \n",
    "        return x\n",
    "\n",
    "# 队列结构\n",
    "class SSRQueue(fluid.dygraph.Layer):\n",
    "    def __init__(self, in_dim, out_dim, stride=1, queues=2, act=None):\n",
    "        \"\"\"\n",
    "        功能:\n",
    "            初始化队列结构，H/W=(H/W+2*P-F)/S+1\n",
    "        输入:\n",
    "            in_dim  - 输入维度\n",
    "            out_dim - 输出维度\n",
    "            stride  - 滑动步长，1保持不变，2下采样\n",
    "            queues  - 队列长度，分割尺度为2^(n-1)\n",
    "            act     - 激活函数\n",
    "        输出:\n",
    "        \"\"\"\n",
    "        super(SSRQueue, self).__init__()\n",
    "        \n",
    "        # 添加队列变量\n",
    "        self.queues = queues # 队列长度\n",
    "        self.split_list = [] # 分割列表\n",
    "        \n",
    "        # 添加队列列表\n",
    "        self.queue_list = [] # 队列列表\n",
    "        for i in range(queues):\n",
    "            # 添加队列项目\n",
    "            queue_item = self.add_sublayer( # 构造队列项目\n",
    "                'queue_' + str(i),\n",
    "                ConvUnit(\n",
    "                    in_dim=(in_dim if i==0 else out_dim), # 每组队列项目除第一个外，in_dim=out_dim\n",
    "                    out_dim=out_dim,\n",
    "                    filter_size=3,\n",
    "                    stride=(stride if i==0 else 1), # 每组队列项目除第一块外，stride=1\n",
    "                    act=act))\n",
    "            self.queue_list.append(queue_item) # 添加队列项目\n",
    "            \n",
    "            # 计算输出维度\n",
    "            if i < (queues-1): # 如果不是最后一项\n",
    "                out_dim = out_dim//2 # 输出维度减半\n",
    "                self.split_list.append(out_dim) # 添加分割列表\n",
    "            \n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        功能:\n",
    "            对输入的特征图像提取特征\n",
    "        输入:\n",
    "            x - 输入特征\n",
    "        输出:\n",
    "            x - 输出特征\n",
    "        \"\"\"\n",
    "        # 提取特征\n",
    "        x_list = [] # 队列输出列表\n",
    "        for i, queue_item in enumerate(self.queue_list):\n",
    "            if i < (self.queues-1): # 如果不是最后一项\n",
    "                x = queue_item(x) # 提取队列特征\n",
    "                x_item, x = fluid.layers.split(input=x, num_or_sections=[-1, self.split_list[i]], dim=1)\n",
    "                x_list.append(x_item) # 添加输出列表\n",
    "            else: # 否则不对特征分割\n",
    "                x = queue_item(x) # 提取队列特征\n",
    "                x_list.append(x) # 添加输出列表\n",
    "        \n",
    "        # 联结特征\n",
    "        x = fluid.layers.concat(input=x_list, axis=1) # 队列输出列表按通道维进行特征联结\n",
    "        \n",
    "        return x\n",
    "\n",
    "# 基础结构\n",
    "class ResBasic(fluid.dygraph.Layer):\n",
    "    def __init__(self, in_dim, out_dim, stride=1, is_pass=True):\n",
    "        \"\"\"\n",
    "        功能:\n",
    "            初始化基础结构，H/W=(H/W+2*P-F)/S+1\n",
    "        输入:\n",
    "            in_dim  - 输入维度\n",
    "            out_dim - 输出维度\n",
    "            stride  - 滑动步长\n",
    "            is_pass - 是否直连\n",
    "        输出:\n",
    "        \"\"\"\n",
    "        super(ResBasic, self).__init__()\n",
    "        \n",
    "        # 是否直连标识\n",
    "        self.is_pass = is_pass\n",
    "        \n",
    "        # 添加投影路径\n",
    "        self.proj = ConvUnit(in_dim=in_dim, out_dim=out_dim, filter_size=1, stride=stride, act=None)\n",
    "        \n",
    "        # 添加卷积路径\n",
    "        self.con1 = ConvUnit(in_dim=in_dim, out_dim=out_dim, filter_size=3, stride=stride, act='relu')\n",
    "        self.con2 = SSRQueue(in_dim=out_dim, out_dim=out_dim, stride=1, queues=3, act='relu')\n",
    "        \n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        功能:\n",
    "            对输入的特征图像提取特征\n",
    "        输入:\n",
    "            x - 输入特征\n",
    "        输出:\n",
    "            x - 输出特征\n",
    "            y - 输出特征\n",
    "        \"\"\"\n",
    "        # 直连路径\n",
    "        if self.is_pass: # 是否直连\n",
    "            x_pass = x\n",
    "        else:            # 否则投影\n",
    "            x_pass = self.proj(x)\n",
    "        \n",
    "        # 卷积路径\n",
    "        x_con1 = self.con1(x)\n",
    "        x_con2 = self.con2(x_con1)\n",
    "        \n",
    "        # 输出特征\n",
    "        x = fluid.layers.elementwise_add(x=x_pass, y=x_con1, act='relu') # 直连路径与卷积路径进行特征相加\n",
    "        \n",
    "        return x\n",
    "    \n",
    "# 模块结构\n",
    "class ResBlock(fluid.dygraph.Layer):\n",
    "    def __init__(self, in_dim, out_dim, stride=1, basics=1):\n",
    "        \"\"\"\n",
    "        功能:\n",
    "            初始化模块结构，H/W=(H/W+2*P-F)/S+1\n",
    "        输入:\n",
    "            in_dim  - 输入维度\n",
    "            out_dim - 输出维度\n",
    "            stride  - 滑动步长\n",
    "            basics  - 基础长度\n",
    "        输出:\n",
    "        \"\"\"\n",
    "        super(ResBlock, self).__init__()\n",
    "        \n",
    "        # 添加模块列表\n",
    "        self.block_list = [] # 模块列表\n",
    "        for i in range(basics):\n",
    "            block_item = self.add_sublayer( # 构造模块项目\n",
    "                'block_' + str(i),\n",
    "                ResBasic(\n",
    "                    in_dim=(in_dim if i==0 else out_dim), # 每组模块项目除第一块外，输入维度=输出维度\n",
    "                    out_dim=out_dim,\n",
    "                    stride=(stride if i==0 else 1), # 每组模块项目除第一块外，stride=1\n",
    "                    is_pass=(False if i==0 else True))) # 每组模块项目除第一块外，is_pass=True\n",
    "            self.block_list.append(block_item) # 添加模块项目\n",
    "    \n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        功能:\n",
    "            对输入的特征图像提取特征\n",
    "        输入:\n",
    "            x      - 输入特征\n",
    "        输出:\n",
    "            x      - 输出特征\n",
    "        \"\"\"\n",
    "        for block_item in self.block_list:\n",
    "            x = block_item(x) # 提取模块特征\n",
    "            \n",
    "        return x\n",
    "\n",
    "# 模组结构\n",
    "class ResGroup(fluid.dygraph.Layer):\n",
    "    def __init__(self):\n",
    "        \"\"\"\n",
    "        功能:\n",
    "            初始化模组结构，H/W=(H/W+2*P-F)/S+1\n",
    "        输入:\n",
    "        输出:\n",
    "        \"\"\"\n",
    "        super(ResGroup, self).__init__()\n",
    "        \n",
    "        # 添加模组列表\n",
    "        self.group_list = [] # 模组列表\n",
    "        for i, block_arch in enumerate(group_arch):\n",
    "            group_item = self.add_sublayer( # 构造模组项目\n",
    "                'group_' + str(i),\n",
    "                ResBlock(\n",
    "                    in_dim=block_arch[0],\n",
    "                    out_dim=block_arch[1],\n",
    "                    stride=block_arch[2],\n",
    "                    basics=block_arch[3]))\n",
    "            self.group_list.append(group_item) # 添加模组项目\n",
    "    \n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        功能:\n",
    "            对输入的特征图像提取特征\n",
    "        输入:\n",
    "            x      - 输入特征\n",
    "        输出:\n",
    "            x      - 输出特征\n",
    "        \"\"\"\n",
    "        for group_item in self.group_list:\n",
    "            x = group_item(x) # 提取模组特征\n",
    "            \n",
    "        return x\n",
    "        \n",
    "# 残差网络\n",
    "class ResNet(fluid.dygraph.Layer):\n",
    "    def __init__(self):\n",
    "        \"\"\"\n",
    "        功能:\n",
    "            初始化残差网络，H/W=(H/W+2*P-F)/S+1\n",
    "        输入:\n",
    "        输出:\n",
    "        \"\"\"\n",
    "        super(ResNet, self).__init__()\n",
    "        \n",
    "        # 添加初始化层\n",
    "        self.conv = ConvUnit(in_dim=3, out_dim=16, filter_size=3, stride=1, act='relu')\n",
    "        \n",
    "        # 添加模组结构\n",
    "        self.backbone = ResGroup() # 输出：N*C*H*W\n",
    "        \n",
    "        # 添加全连接层\n",
    "        self.pool = Pool2D(global_pooling=True, pool_type='avg') # 输出：N*C*1*1\n",
    "        \n",
    "        stdv = 1.0/(math.sqrt(group_dim)*1.0)                    # 设置均匀分布权重方差\n",
    "        self.fc = Linear(                                        # 输出：=N*10\n",
    "            input_dim=group_dim,\n",
    "            output_dim=class_dim,\n",
    "            param_attr=fluid.initializer.Uniform(-stdv, stdv),   # 使用均匀分布初始权重\n",
    "            bias_attr=fluid.initializer.Constant(0.0),           # 使用常量数值初始偏置\n",
    "            act='softmax')\n",
    "    \n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        功能:\n",
    "            对输入图像进行分类\n",
    "        输入:\n",
    "            x - 输入图像\n",
    "        输出:\n",
    "            x - 预测结果\n",
    "        \"\"\"\n",
    "        # 提取特征\n",
    "        x = self.conv(x)\n",
    "        x = self.backbone(x)\n",
    "        \n",
    "        # 进行预测\n",
    "        x = self.pool(x)\n",
    "        x = fluid.layers.reshape(x, [x.shape[0], -1])\n",
    "        x = self.fc(x)\n",
    "        \n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tatol param: 2116538\n",
      "infer shape: [1, 10]\n"
     ]
    }
   ],
   "source": [
    "import paddle.fluid as fluid\n",
    "from paddle.fluid.dygraph.base import to_variable\n",
    "import numpy as np\n",
    "\n",
    "with fluid.dygraph.guard():\n",
    "    # 输入数据\n",
    "    x = np.random.randn(1, 3, 32, 32).astype(np.float32)\n",
    "    x = to_variable(x)\n",
    "    \n",
    "    # 进行预测\n",
    "    backbone = ResNet() # 设置网络\n",
    "    \n",
    "    infer = backbone(x) # 进行预测\n",
    "    \n",
    "    # 显示结果\n",
    "    parameters = 0\n",
    "    for p in backbone.parameters():\n",
    "        parameters += np.prod(p.shape) # 统计参数\n",
    "    \n",
    "    print('tatol param:', parameters)\n",
    "    print('infer shape:', infer.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "### 训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xl8VNX9//HXmSUrSQghECJbAEUimxCRRRBEcUFFW6pS3Gv5Vvuz4tLWpV+VViu1tVW/aq2i1rpAFUXqiktBcGOVTRaBABLWJJB9m+X8/pghRiDJAAmZGd7PxyMPZu7cufdzc4d3zpx77r3GWouIiEQ+R0sXICIiTUOBLiISJRToIiJRQoEuIhIlFOgiIlFCgS4iEiUU6CIiUUKBLiISJRToIiJRwtUcC23btq3t2rVrcyxaRCQqLV26tMBam340y2iWQO/atStLlixpjkWLiEQlY8zWo12GulxERKKEAl1EJEoo0EVEokSz9KEfisfjIS8vj6qqqmO1yqgVFxdHx44dcbvdLV2KiISRYxboeXl5JCUl0bVrV4wxx2q1UcdaS2FhIXl5eWRlZbV0OSISRo5Zl0tVVRVpaWkK86NkjCEtLU3fdETkIMe0D11h3jT0exSRQwn/g6LWQkUhWH9LVyIiEtbCP9C9VVD0HVSXHvEiCgsL6d+/P/379ycjI4MTTjih9nlNTU1Iy7juuutYv359yOucNm0akydPPtKSRUQO2zE7KHrE9t/E+ihuZp2Wlsby5csBuP/++2nVqhV33HHHAauxWGtxOA79N+6FF1444vWLiBwL4d9Cxx7wb9PZuHEj2dnZTJw4kVNOOYWdO3cyadIkcnJyOOWUU/j9739fO+8ZZ5zB8uXL8Xq9tG7dmjvvvJN+/foxZMgQ9uzZ0+B6Nm/ezKhRo+jbty/nnHMOeXl5AMyYMYPevXvTr18/Ro0aBcCqVas47bTT6N+/P3379iU3N7fJt1tEolOLtNCnvP0Na3aUhDaz9YGnElxl4Ki/3OzMZO676JTDrmXdunX861//IicnB4CpU6fSpk0bvF4vo0aNYvz48WRnZ//gPcXFxZx55plMnTqV2267jeeff54777yz3nXcdNNN3HDDDUycOJFnnnmGyZMnM3PmTKZMmcK8efNo3749RUVFADz11FPccccdXH755VRXV2OP4puJiBxfIqCF3ry6d+9eG+YA06dPZ8CAAQwYMIC1a9eyZs2ag94THx/P+eefD8DAgQPZsmVLg+tYuHAhV1xxBQBXX301CxYsAGDYsGFcffXVTJs2Db8/cNB36NChPPDAAzz88MNs27aNuLi4pthMETkOtEgL/bBa0tVlULgBWneBhDZNXktiYmLt4w0bNvDYY4+xaNEiWrduzZVXXnnI8d4xMTG1j51OJ16v94jW/eyzz7Jw4ULeeecdBgwYwNdff81VV13FkCFDePfddznvvPN4/vnnGTFixBEtX0SOL8d9C72ukpISkpKSSE5OZufOncyZM6dJljt48GBee+01AF5++eXagM7NzWXw4MH84Q9/IDU1le3bt5Obm0uPHj245ZZbuPDCC1m5cmWT1CAi0S/8R7kcQwMGDCA7O5uTTz6ZLl26MGzYsCZZ7pNPPsn111/PQw89RPv27WtHzNx6661s3rwZay1jxoyhd+/ePPDAA0yfPh23201mZib3339/k9QgItHPNMdBt5ycHHvgDS7Wrl1Lr169Dn9h1aVQuBFad4aEtCaqMPId8e9TRMKSMWaptTan8Tnrpy4XEZEoEf6BXntiUcuWISIS7sI/0Gsp0UVEGhJBgS4iIg2JoEBXC11EpCEREOgKchGRUERAoDeNUaNGHXSi0KOPPsqNN97Y4PtatWoFwI4dOxg/fvwh5xk5ciQHDtNsaLqISHMIKdCNMbcaY74xxqw2xkw3xhy7C4w00SiXCRMmMGPGjB9MmzFjBhMmTAjp/ZmZmcycOfPoihARaUaNBrox5gTgV0COtbY34ASuaO7CDnZ0iT5+/Hjefffd2htabNmyhR07djB8+HDKysoYPXo0AwYMoE+fPsyePfug92/ZsoXevXsDUFlZyRVXXEGvXr249NJLqaysbHT906dPp0+fPvTu3Zvf/va3APh8Pq699lp69+5Nnz59+Nvf/gbA448/TnZ2Nn379q29qJeISGNCPfXfBcQbYzxAArDjqNb6/p2wa1Vo8/q94K0EZyw4Y+qfL6MPnD+13pfbtGnDoEGDeP/99xk3bhwzZszgsssuwxhDXFwcs2bNIjk5mYKCAgYPHszFF19c7707//73v5OQkMDatWtZuXIlAwYMaHATduzYwW9/+1uWLl1KamoqY8aM4a233qJTp05s376d1atXA9ReQnfq1Kls3ryZ2NjY2mkiIo1ptIVurd0O/AX4DtgJFFtrP2zuwupU0GRLqtvtUre7xVrL3XffTd++fTn77LPZvn07u3fvrnc58+fP58orrwSgb9++9O3bt8H1Ll68mJEjR5Keno7L5WLixInMnz+fbt26kZuby80338wHH3xAcnJy7TInTpzIyy+/jMuly+2ISGgaTQtjTCowDsgCioDXjTFXWmtfPmC+ScAkgM6dOze80AZa0gep3Af7tkByJrRqH/r7DmHcuHHceuutLFu2jIqKCgYOHAjAK6+8Qn5+PkuXLsXtdtO1a9dDXja3qaWmprJixQrmzJnD008/zWuvvcbzzz/Pu+++y/z583n77bd58MEHWbVqlYJdRBoVykHRs4HN1tp8a60HeBMYeuBM1tpnrLU51tqc9PT0pquwCU/9b9WqFaNGjeL666//wcHQ4uJi2rVrh9vtZu7cuWzdurXB5YwYMYJXX30VgNWrVzd6idtBgwbx6aefUlBQgM/nY/r06Zx55pkUFBTg9/v58Y9/zAMPPMCyZcvw+/1s27aNUaNG8ac//Yni4mLKysqOfuNFJOqF0uz7DhhsjEkAKoHRQAuMxWuarpcJEyZw6aWX/mDEy8SJE7nooovo06cPOTk5nHzyyQ0u48Ybb+S6666jV69e9OrVq7alX58OHTowdepURo0ahbWWsWPHMm7cOFasWMF1111Xe7eihx56CJ/Px5VXXklxcTHWWn71q1/RunXro99wEYl6IV0+1xgzBbgc8AJfAzdYa6vrm79JL59bsReKtkJSB0jKOPz3RyldPlckujTF5XND6pi11t4H3Hc0KzpyOlNURCQUx82ZoiIi0e6YBvoR3R3JHvTguNccd5kSkch3zAI9Li6OwsLCIwgj3eCiLmsthYWFxMUdu6sviEhkOGaDmzt27EheXh75+fmH98bqMqjcC3HVEKezJiHwx7Fjx44tXYaIhJljFuhut5usrKzDf+Pi52DObTD8dhh9b9MXJiISJSLgoOj+Lhd/y5YhIhLmwj/Qa88UVSe6iEhDIiDQ/T/8V0REDkmBLiISJSIo0NXlIiLSkAgKdLXQRUQaEjmBrjOLREQaFDmBrha6iEiDIiDQNQ5dRCQUERDoaqGLiIQiAgJdLXQRkVBEQKBr2KKISCgiKNDVQhcRaUgEBbpa6CIiDYmgQFcLXUSkIQp0EZEooUAXEYkS4R/otaf8qw9dRKQh4R/oaqGLiIQkAgJdJxaJiIQiAgJdLXQRkVAo0EVEokQEBboOioqINESBLiISJSIo0NXlIiLSkAgIdI1yEREJRQQEulroIiKhUKCLiESJCAh0nfovIhKKCAh0tdBFREKhQBcRiRIRFOjqchERaUhIgW6MaW2MmWmMWWeMWWuMGdLchdVSC11EJCSuEOd7DPjAWjveGBMDJDRjTQfYPw5dLXQRkYY0GujGmBRgBHAtgLW2Bqhp3rLqUAtdRCQkoXS5ZAH5wAvGmK+NMdOMMYnNXNf3FOgiIiEJJdBdwADg79baU4Fy4M4DZzLGTDLGLDHGLMnPz2+6ChXoIiIhCSXQ84A8a+3C4POZBAL+B6y1z1hrc6y1Oenp6U1Xoa7lIiISkkYD3Vq7C9hmjOkZnDQaWNOsVf2ggP1BroOiIiINCXWUy83AK8ERLrnAdc1X0gHU5SIiEpKQAt1auxzIaeZa6lm5Al1EJBQ6U1REJEpEQKDroKiISCgiINDV5SIiEooICnR1uYiINCSCAl0tdBGRhkRAoKsPXUQkFBEQ6Gqhi4iEQoEuIhIlIifQdeq/iEiDwj/QdYMLEZGQhH+gq8tFRCQkCnQRkSihQBcRiRIKdBGRKBEBga6DoiIioYiAQFcLXUQkFAp0EZEoEQGBri4XEZFQRECgq4UuIhKKyAl0nfovItKgyAl0tdBFRBqkQBcRiRIREOi6wYWISCgiINDVQhcRCUUEBboOioqINCSCAl0tdBGRhoR/oNcOV7RqpYuINCD8A71uy1yBLiJSrwgLdHW7iIjUR4EuIhIlIivQdfq/iEi9IiDQ64S4WugiIvWKgEBXl4uISCgU6CIiUSICAt2CwxV8rEAXEalPBAS6v06g66CoiEh9IizQ1UIXEalPyIFujHEaY742xrzTnAUdxPrBOIOP1UIXEanP4bTQbwHWNlch9bJ+cDi/fywiIocUUqAbYzoCY4FpzVvOAawFdFBURCQUobbQHwV+AxzbRN3fxaIWuohIoxoNdGPMhcAea+3SRuabZIxZYoxZkp+f3zTV7Q/w/S10nfovIlKvUFrow4CLjTFbgBnAWcaYlw+cyVr7jLU2x1qbk56e3kTlBQPcBMtUC11EpF6NBrq19i5rbUdrbVfgCuC/1torm70yOLiFrkAXEalXeI9DV6CLiITM1fgs37PWzgPmNUslh1yhAl1EJFQR0kLXiUUiIo1RoIuIRIkICXR1uYiINCbMA33/sEWdWCQi0pgwD3S10EVEQhXmgX7Aqf86U1REpF5hHugHHhRVC11EpD4REujqchERaYwCXUQkSkRIoKvLRUSkMRES6LpJtIhIYyIj0HVPURGRRkVGoKvLRUSkUeEd6PvpoKiISKPCO9DVQhcRCVmEBLpa6CIijYmMQDc69V9EpDGREehqoYuINCpCAl196CIijYmQQFcLXUSkMREW6OpDFxGpT5gH+gHXQ1cLXUSkXpER6Dr1X0SkUWEe6DooKiISqrAK9MKyah58dw0eXzC4dVBURCRkYRXoD72/jmcXbObGl5cGJijQRURCFlaB7g/2kX+8dk9gggJdRCRkYRXocW7nDycc2IeuU/9FROoVXoHuaiTQ1UIXEalX2AS6z295/vPNtc+ttTqxSETkMIRNoDsdpvZxOkVUlOyltovFqIUuItKYsAn0uv4Z8yccH96tg6IiIochrAJ9cLc2uPByksnDlGw/xKn/6nIREalPWAX6qzcM5u9j2+I2Pqjcp4OiIiKHIawC3eEwpFZ+F3hcXawuFxGRwxBWgQ6QXLEVAKcCXUTksIRdoCeVBwO9pgR8nsBEBbqISKMaDXRjTCdjzFxjzBpjzDfGmFuas6CE0u/HolNVFCwiWKYCXUSkXq4Q5vECt1trlxljkoClxpiPrLVrmqOg+NItVFs3scYDFXsDE3Xqv4hIoxptoVtrd1prlwUflwJrgROapZqacmLKd/KN7RJ4XlEYrFJnioqINOaw+tCNMV2BU4GFzVEMwO5Rf+EN34jAk8p9gX/Vhy4i0qiQA90Y0wp4A5hsrS05xOuTjDFLjDFL8vPzj6yamESqe0/kK3+vwPPaLhcFuohIY0IKdGOMm0CYv2KtffNQ81hrn7HW5lhrc9LT04+4IL+1lNjEwJP9XS46KCoi0qhQRrkY4DlgrbX2r81dUJe0BIoJBnrlgS109aGLiNQnlBb6MOAq4CxjzPLgzwXNVZAxhlvO7U2FjcVXfuBBUbXQRUTq0+iwRWvtZ4BpbL6mdNbJ7Siel0iHKvWhi4iEKuzOFAXolp5I0f5+dNDFuUREQhCWgR7rclKCAl1E5HCEZaADmOBdijzWCTGtAhN1UFREpF5hG+gxXQcBcKvnJohPDU5VoIuI1CdsA31x1i84qepF3vEP0Th0EZEQhG2gD+7RgRrcADz16abARAW6iEi9wjbQ+3RM4exe7QB4+IP1gFGgi4g0IGwDHWBQVhsA4t3OQLeLAl1EpF5hHeg3nNENh4HEWBfW4QJvdUuXJCIStsI60B0Ow4V9Mykoq+ZrTyfyVvyX2cu3t3RZIiJhKawDHaBtq1gA5vn6k1m+likz5rdwRSIi4SnsA73S4wNgnr8fDmMZ4VjZwhWJiISnsA/0rmkJAKyyWRTYZEY6l7dwRSIi4SnsA/1nZ2Tx6s9Px+Jgsb8nOa7NLV2SiEhYCvtAdzkdnNY1MHxxnb8zmXYX1JS3cFUiIuEn7AMdwO0MlLnOdsaBZdx905i9fDvvr9r5/Uw+DxR910IVioi0vEZvcBFO1trOAJzpWMmemV/xiPcnzMruTP/Orbm0YhYZyx7B3PEtxCW3cKUiIsdeRAX6NptOuY3lZtcs3MZHru3A9DWj+XDNbk5yv0sHZyXsWQudT2/pUkVEjrmI6HLZL87tZr3thNsEhjL+1PlJ8BXLqY6NAPh3r2mh6kREWlbEBPqy/z2HhfeMZp0/0O3yT+8Y+ji20Mfk0tnsIc2UAlC+TePUReT4FDFdLm0SYwA4deIUPlh9Po70wXgXLODO9C94raArAPtsK/w7VrVglSIiLSdiAn2/Xr360qtX38CTkp8waPlrFDv7Umbj+NCXw5j8Jdz1xgqMw8HcdXv496QhdA6enCQiEs0ipsvlkAZeh9tfxQXORczyncEa24VUU8Z/F6/k1YXfsbO4ihF/ngvApvwySqs8tW8tqfJQ4w3hcrzVpVC0rbm2QESkyUR2oJ8wAE69ise8l3Kv91ruv+laAF6L+T29zNba2T5es5vRj3zKJU9+zqq8Yv7nX4sZO+Ulxj7wb95btROvz09hWTXVXt/B6/joPnh2FPgP8VpTKM6Df10CZfnNs3wROW5EdqADjHuCHpc9xG/Oy8acMIC3sh8jxZTzS9dsAJIp45mXXgIsQ/fO4pGnnmDchrtZEHsrc+yN/N+rs/jDO2sY+MDHjHh47sHLz50H5fmB4ZAH+GZHMY99vAGAyhoffv8R3MR6/fuQOxc2f3r47xURqSPi+tAPZWzfDrWPL7nsWipmfc2YldNZMOxb0r6aSgJVfO3vUTu0EeBp74Xc4HyPsc6v+MuXXcigkKTSSnr+zsMFfTrwyE/6UVW0g4S9gfuZbln+CV3P6/2D9V7y5Od4fJbrzuhK3/s/5IYzsvjdhdnfz+CpAlcsGFN/8XlLAv/u/gb6jD/6X4aIHLciv4V+CAkDLifGVtPpq/uJ6TSA6b5RnOrYyFv2TP7gmchdnp8x1ftTFvp7ca4jEKhPxPwfr8Y8iNfrYdbX2+l293s8+NTzAHisk5Wfv1/bAt9XXoPfb/H4As8/WbsbgGmfbcbrC/TLFy+ajn04i9z/TGXaglw+/TbQpfL2ih30vX8OVcHLArN9aeDfPYc3fr7a66OqpBBKd9c7T2FZdW09h5S/HsoLDmu9YWHb4sA3GxH5gahooR+k02BI7gjGgWvCK0yIT4VdK/nkkxreXr2Hfh1T+J9uaaTWXMqJyx/gIscX5Di+BWCYYzWFNoX2Zi/DqudT4YjlU39fhjjW8MS91/KK92x204Zx/TNrV3frv1fUPu5xz/v882zLyM9+gd8aqpe+ygM1gVE5z1w1kDvfWEl5jY+ShS8zZ+lSxu0LdNmw+5vv6/f7wOE85Ka9sTSP7u1acfP0ZUwpf5CzMqrhxs8Omq/G62fgAx8zYVBnHrqgM8S0ql3mjqJK3MaS/sIFkN4Trn234W8R4ebj++C7L2Hi69Dj7JauRiRsGGuPoN+3ETk5OXbJkiVNvtzDUrABXHHQulPtpJIqD/PW53Nxv2AYl+zA97e+WL8P63RT6XNQTjwdzN7a98zz9eNj/wAecL8AwEp/Fg97r8Bgybet6ePIJYlKLnR+iQPLG77h9HPkcq5jMc94x3K7eybDq//GNtsegAwKyTD7eDPu9zisF4AFvt4Md67myx8tpHzmTeSklLDpkreZtaqAn6cup0v/UZDSkZV5RVz8xOcAOPGxIvbntDJVeG/fRG5FLCe1T6Kixsv2fZXEuZ0Mf3guLrysb3M7uzNGknn1NPaUVjHowU/obzbyVuy9gY28ahZ0Pyvw2Fowhllf5/Hk3E3MmTwCp+PgsLfWUlHjIzHWRUFZNff95xv+eGkfUuLdB807Y9F3pCfFMrpX+6Pbp/vrm9oZqksgrjXcuhpik45+uQ3xeQLrdcU073rkuGaMWWqtzTmaZURnCx2g7YkHTUqOc38f5gDJmTivnoWdNQmTfQkfL1nP+d5PmO0byvPe84jFw0Z7AmXEs8Om8ZM+rTl//e94Oeahg5btSevJmnwPf3D/E4918rrvTGb7h3E7MxnjWMpzvguY5Hybu93TAcj3J5NrMznVbOB135kMd64mZeZlDHFshVJ47/kHKLXJdIl5ivc/Po2qCx6n5N17+Sz2aybU3EMqZbQyVQD8/KGn+dzfG4PFgws/Dsb2bkcns5sO7MVZkU9m7ut88OZQfrmoLeDkTMcKLIZiVxru//wG33XvM/fbvQz74ALSYi2FpYPI9U4kv7SajJQ4IHDgd19FDcmmkpWfvMLVizqz9LzvqPhsGuOrU5jZ8Tl+NjCVb8tiWfHdXn6S05lqn5873wyc7LVl6ljy9lWwq7iKhBgX2ZkHX0Rtb3kNT87dyK/P7Umc+xDfUoq2BsK8/0RY/gqsfgMGXnvQbFUeH+6NH+DcuwlOPAfa9fr+RWuhohAb04qVu6rolxEHb90IfS+HnucdvM7XrwVPReAPXx2FZdX4rKVdUtzB7xFpAdHbQj8c1oK1VBVuw79qJiX9J+F3uIhxOdhXXkNGShxfbCrk3FMyuO3Bv7C3vIrJ5/RiwbJV9B5yLpnpbejZ4yTO++t/+X3xXQxyrGdc9e9ZYXvwbsxddDT5rPB3Z4RzFR/4TmObTecT/wBW+Lvx1hUZXDtjE1/G3UyNdfKA90rOdizjdMc6ABz4scBi/8kMdqzBYnjFN5o8m8497lfxWcMify/6OjaRaKoBeMl7NrttKre6ZvKpvx8jHCvZadPo5Mhni789l9T8nudi/oIDyyPen/Cc+8+ss52Z7RvGve6XmOfrx0jnCt72DWbf0Hu4aN+LvM1I7l3ZhlZU8FLMVE51bOQf3rH83PUem/0ZdHfsZJG/J4Mc6/md5zrGOT8nNTmJCwp+RQrlXOz8gvXtL+CzHQCW3mYzrwzdjX/zApYljeTGTaeT3iqOaq+fgrJq3E7DB5NH0LlNAos27+Wk9knExzhZNuclRiybzOKzXydnxb1sLbFMy3qUKZcNZvQj83iw/z6GlX7AyMWD+CT21zjxU2hTWH/5AoZmd6Fq307Kpl1I2/KN7KIt46v/lxcGbefEFQ9jY5P5dNSbDB+UU/utpKp0L3F/OxH8XrhtLTapA19uKiSneA7XvpHHF/7eLLpnNJvzyxnQJbX2Us/1+uYt+OheSOsO2ZfAKZeGdnXQb2ZB2omQ0ZvSKg8FJZVkrXoMsi9mfmkmg7ulEeP6ft0+v6XG6yc+5tBddxJ+mqKFrkA/TPvKayiq9JDVNvGg10Y/Mo+C/N18fHkrrv8shXZJsfgLc3m61bO4CtaxOut6ul96D/nlXmYv386grDYM7d6Wrne+y8WOL1hrOzPyjOHEVezgV+7ZzFyyjTn+HP4Z82cAHvJMoLvZwSXur9gd35127krW7jP0d+Syy7TjnzVn0duxhQudX1FtXcSaQJfOIn9PfllzC2OcS7jP9SJrbFf6ODbzhPcS/uYdz9mOpUyLeQSfNayznRlb80cmOd/hbvd0aqyTGOPDbw1v+IbT37GJrmYXu2wbOjnyqbFOhlU/zh/dz3GOcxn7bCtSTVnt72STvwOZppB4U8MCX28+8/fhKtdHdDQFeK2DrbY93R07ecM3nBneUVzj+hAvDtpQSi/Hd6zxd+Fmz82UEPh9T3bN5FfOWZxS/RxXOOdyn/slINBtNc/fj9tdM0kw1Wz0Z9LDsYPban7BX2Oe5jHvjzhlzHVkfz6Z1lV5POkdxyTXO9TgJoEq1tgu9HFtx+vzsShuGKec9zM2JAzk9Rcf47GYpwD4b9YdeAfewNoZd3OL600AXvSewxz/aWzwn0BbU8Kfuq/mzc1OfnLW6bRf/gT55zzOSb1Po6TSw7TnnuLWwinkx3UhI9GJ2bsRjyOOBTFn8JbjHHqedjZff7eP64d0pl1KPJ+sy2dnUSW/6m9I/ecZ2Ix+OCbN5ay/fkqvwk94MuZxapI60T9/CuOH9OS2c06idUIMi7fs5YXPN/Peql1sfugCjDGs3VnC/G/z+Z+hJ/D+ym2U+GPplJrA0B5tD/ocV3l8xLocFFd6MBhSEg7uRttv+Xf7ePHLrfx5fF9cjf0xkwYp0MPM+l2lvLksjzvPPxlT9yCjteCrCQxhPISNe8oor/YyY/E2fnNuT1KD163ZlF9GSrwbx7OjKNhXxL8HvkrvuHwu+epyjPXBoEnsLK2hw9p/svu8Z5hediodWzk59b2xdDZ7mO0fxnjnfNacfDO/3nMuXdISuMk5m97rHsNmjWRm1hTeWFfJV7l7udP1Kr9wvcNdnp8x3TcagEdPXEH21pe5z3stYxxLuMI1jzIbyy2e/8fYXqlM3PRrXvOeyZud72JN7jZOd6xlre3MOzH38JZvGBvtCfzYuYCV/iwKbQq3uWcCgfD9j38oH/kGUkQrJrveYHIwIAttEmU2niJasc22Y4xjMRXEsdum8qZvOCMcK8kwexld8wjxVPFz53u4jZdrnB+SbCpY7+9IJTH0d+SyyN+Ty2ru4x/uv3KuM/B5LLYJ/NJzC5/5+9Db5HKz6y0yzF5u9dxEHDVc4/yQ852LSDYV7LGtKbXxJJlKimwiVcSwMfFUflT5Jq95z8SHg/HO+bVX/wTwW4PDWHzW4DSWbf50/uG7kFzbgSfcj7PdtuXymnupIJb+ZhOXOedxkfNLkkwlM7wjGejYQDezg70kUWBb09nsZqvNINsROFHuFeclJNfsoqfZRmtTTluK+dr24F3fYEqJZ7aFajWYAAAMEklEQVRvGJmmgGKbyD6SWXxdGxbNmcGLOzvRzhRxb+x0vH4/E2vuoZ/ZxGDnOrzdRpE5+DKuf3Epg7qm4tjzDX2ql7HFZrDKn0W/zHji07NYsXI5v+u5nZHDhnPVvATOcq3i0i1TeMJ7CT0uuoOJvVys2bCBJ5dVs3Kfm5k3DuPb3aUM7JJKQowLrMW37zs2ViaR1b41jqItONe/g/fUayhZ/G/STj4D2n8/7LfK4+PXM1dyy+ge9GiXxNbCch79eAM3DM/ilMwUPvxmF8YYzsluT83ePF7+poafDu4CQG5+OdmZyXh8fq5+bhF9OqZw9wWBbjef37J+V+khu/xakgL9eFGxl2qfJaZVm8Afin1bA326aT2gZAds+BBOu6F2pErBtvVs2rCWQUNHYT78Xzjzt/iSMgPdCNYGhium96yd/4tNBSTHwJ///gxDx4ynS9skzsnOwOkwtQdifz48i/83LJN5G/fy1qoCXrj2NArmP0tVt3Pp2Cnwn6ja68NaiLPVbNznpX1KPElxbr7dXcp/1+7mGv8sfv1xMe/4BwOGGZMGs3p7MT3bt+K0Lf9gxeL53FR+A2/efiEOY/hwzW7OdK+ldOFL1BTk1nZD2VN+xG+YzNsrd9A1LZF1u0pJpJIEqkjP6ETnoq94mj+yZdifuH7FyewuKOAS5xe0ppSZdhS7/SkApCa4Kany4jvghLBYahjpWM4VzrmMcq7gBe+5bLEZTHG/CMCr3lHc4/0ZFgetqGCAYwMnOndT6EvgC/8p/Mz1Hv0cuTzpHcfT8U+R6CsJ/H6sm7E1D7LRdvzB+uKp4l7XS0xwzWWrqyuzq/rTzeyirSnGYDndsY5/ec9hrPMr0kwpHuvEbXz82jMJg2Wy6w0ygwfy99jWtDNF+KxhL8mkm+IfrGuTvwPppohkU1lbU6zxsMF/Avk2hSzHrh8MCthv/x+o/T73nUIfRy5O/CSa6oNer7QxbLdtWeY/kWIS6Wm20cVZSBd2sMbfhRd9Y7glWHeFjSXBVFNNDF+lXcKXuxy0T4phRvUQqqsq6G820d+xkdamjE98A1hvO5GY2Ytvt+fT3ezgx875XOP6iK/9PZjv78sWf3vKieNH6dt5Lb8ziVRRSDImoQ09Y/aS3iaF/24qp2+3DqQkp7Cq0DB+RH8+Xb+H4ooaHJ4KToirYsN32yguLsHEp5LVuTML8nwM7ZFGe1NE5/gqKhI7stfj5qUvNvPHH59Ktc+SnZnMyRlH9odCgS7HRJXHR4zTgeMQo10OV3GFh9U7isnpmkqs6zD7d/PXw+b5kHUmpJ9UO/mbHcWc1D6ptv/a77c4dq2AjL7gcFBUUVM7+sYYg99vKa3ykpLgxlrLzuIqVm8vJs7tpF+n1iTEOCmq8BDrdpBckQet2rOp2M8Lsz9iYFsvOcPPx+l0snxbEef3zqj9NvbFpgK6piWS2TqeveU17CyuJLt9In9+41PcW+fjb5XBuB9fydT31zGsR1t+enpnnMawr8LDvvJq3Hu/pX23Pjzy8SYqPT56Z6aQnZHI268/R0nmcBK3L8BVtIXCE39CjnMj/u6jSU6IZeGmPSxYuZH/O8ND2qpnmVNxEie2jaNq305WVrVjZvXpDI/PJaXtCexMOoWL2myn63dvMt0zgld3dmCi8xOGO1bS2pSz07ahY85YijKGM3vO+2Swj3IvdPDvojKxE++UdOMix5ec51xMDS7udt7OL7tuZ/OG1eTb1uywaXQwhXQ0BXQxuxniWEMsHtbZThQ7Uljk6cFVro9ob4ooson8xXsZE2I+4+WaEYx2LGOEYyUx5uDLbJTbWCqJpa0J/HEss3HE4qn9dvSWbygDzQY6Ob6/hMb+b0tNxW8D+/lQy9z/B/Sn/inM+t1VtIo9/PEmCnQRaRJ+v8UYfthVWIe1tvY1v9+yubCcjqnx+PyWhBgXZdVeEmOcVHn8xLoCf/zzS6tJclv2VVQTFxtP6wQ3BWU1pMWBKcmjxJFCSmrb2mWuyCuie9t4ikpK6RRbTuWSV6FVe5ydTsPZvhcenyW+cBXVu7+lZstC/O5E4rKGsMuZgWmXTWm1hxgHxBR8Q3VJPtsT+9CjZi1xqe2heAfJzmpWVqbh8ntIcXlweitJcFTjrilm756dJMa6qPT4KfHFEJfcFn9sCmU2BiqL6BhTTryniEqPF09CBlsrYuji2ENVZQUulwunrxpv6R7ycu5mRN/uR7QPFOgiIlGiKQJdh6VFRKJESIFujDnPGLPeGLPRGHNncxclIiKHr9FAN8Y4gSeB84FsYIIxJrvhd4mIyLEWSgt9ELDRWptrra0BZgDjmrcsERE5XKEE+glA3Xuw5QWn/YAxZpIxZokxZkl+vu6+IyJyrDXZQVFr7TPW2hxrbU56enpTLVZEREIUSqBvBzrVed4xOE1ERMJIKIG+GDjRGJNljIkBrgD+07xliYjI4QrpxCJjzAXAo4ATeN5a+2Aj8+cDW4+gnrZABN4TrUkcr9uu7T6+aLvr18Vae1T91c1ypuiRMsYsOdozpSLV8brt2u7ji7a7eelMURGRKKFAFxGJEuEW6M+0dAEt6Hjddm338UXb3YzCqg9dRESOXLi10EVE5AiFTaBHwxUdjTGdjDFzjTFrjDHfGGNuCU5vY4z5yBizIfhvanC6McY8HtzmlcaYAXWWdU1w/g3GmGvqTB9ojFkVfM/jpr47ErQAY4zTGPO1Mead4PMsY8zCYK3/Dp7HgDEmNvh8Y/D1rnWWcVdw+npjzLl1pofl58MY09oYM9MYs84Ys9YYM+R42N/GmFuDn/HVxpjpxpi4aN3fxpjnjTF7jDGr60xr9n1c3zoaZK1t8R8C49s3Ad2AGGAFkN3SdR3BdnQABgQfJwHfErhC5cPAncHpdwJ/Cj6+AHgfMMBgYGFwehsgN/hvavBxavC1RcF5TfC957f0dtfZ/tuAV4F3gs9fA64IPn4auDH4+Cbg6eDjK4B/Bx9nB/d9LJAV/Ew4w/nzAbwI3BB8HAO0jvb9TeBaTpuB+Dr7+dpo3d/ACGAAsLrOtGbfx/Wto8FaW/rDESx2CDCnzvO7gLtauq4m2K7ZwDnAeqBDcFoHYH3w8T+ACXXmXx98fQLwjzrT/xGc1gFYV2f6D+Zr4W3tCHwCnAW8E/xwFgCuA/cxMAcYEnzsCs5nDtzv++cL188HkBIMNnPA9Kje33x/wb42wf33DnBuNO9voCs/DPRm38f1raOhn3Dpcgnpio6RJPi18lRgIdDeWrsz+NIuoH3wcX3b3dD0vENMDwePAr8B/MHnaUCRtdYbfF631trtC75eHJz/cH8fLS0LyAdeCHY1TTPGJBLl+9taux34C/AdsJPA/ltK9O/vuo7FPq5vHfUKl0CPKsaYVsAbwGRrbUnd12zgz21UDS0yxlwI7LHWLm3pWo4xF4Gv4n+31p4KlBP4alwrSvd3KoF7ImQBmUAicF6LFtWCjsU+DnUd4RLoUXNFR2OMm0CYv2KtfTM4ebcxpkPw9Q7AnuD0+ra7oekdDzG9pQ0DLjbGbCFwA5SzgMeA1sYYV3CeurXWbl/w9RSgkMP/fbS0PCDPWrsw+HwmgYCP9v19NrDZWptvrfUAbxL4DET7/q7rWOzj+tZRr3AJ9Ki4omPw6PRzwFpr7V/rvPQfYP9R7WsI9K3vn3518Mj4YKA4+BVrDjDGGJMabA2NIdCnuBMoMcYMDq7r6jrLajHW2rustR2ttV0J7Lv/WmsnAnOB8cHZDtzu/b+P8cH5bXD6FcFREVnAiQQOGIXl58NauwvYZozpGZw0GlhDlO9vAl0tg40xCcG69m93VO/vAxyLfVzfOurXkgcaDjjocAGBUSGbgHtaup4j3IYzCHwtWgksD/5cQKC/8BNgA/Ax0CY4vyFwv9ZNwCogp86yrgc2Bn+uqzM9B1gdfM8THHBArqV/gJF8P8qlG4H/oBuB14HY4PS44PONwde71Xn/PcFtW0+dER3h+vkA+gNLgvv8LQIjGKJ+fwNTgHXB2l4iMFIlKvc3MJ3AsQIPgW9lPzsW+7i+dTT0ozNFRUSiRLh0uYiIyFFSoIuIRAkFuohIlFCgi4hECQW6iEiUUKCLiEQJBbqISJRQoIuIRIn/D2lJXhoWwVRIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration:    200, epoch:   1, train loss: 2.378920, valid loss: 0.000000, valid accuracy: 0.00%\n",
      "iteration:    300, epoch:   1, train loss: 2.695031, valid loss: 0.000000, valid accuracy: 0.00%\n",
      "iteration:    400, epoch:   2, train loss: 2.524811, valid loss: 2.062179, valid accuracy: 23.31%\n",
      "iteration:    500, epoch:   2, train loss: 2.652664, valid loss: 2.062179, valid accuracy: 23.31%\n",
      "iteration:    600, epoch:   2, train loss: 2.420298, valid loss: 2.062179, valid accuracy: 23.31%\n",
      "iteration:    700, epoch:   2, train loss: 1.921101, valid loss: 2.062179, valid accuracy: 23.31%\n",
      "iteration:    800, epoch:   3, train loss: 2.047899, valid loss: 2.417096, valid accuracy: 28.45%\n",
      "iteration:    900, epoch:   3, train loss: 1.844729, valid loss: 2.417096, valid accuracy: 28.45%\n",
      "iteration:   1000, epoch:   3, train loss: 1.825090, valid loss: 2.417096, valid accuracy: 28.45%\n",
      "iteration:   1100, epoch:   3, train loss: 1.697457, valid loss: 2.417096, valid accuracy: 28.45%\n",
      "iteration:   1200, epoch:   4, train loss: 1.752214, valid loss: 1.638797, valid accuracy: 43.81%\n",
      "iteration:   1300, epoch:   4, train loss: 1.555009, valid loss: 1.638797, valid accuracy: 43.81%\n",
      "iteration:   1400, epoch:   4, train loss: 1.730682, valid loss: 1.638797, valid accuracy: 43.81%\n",
      "iteration:   1500, epoch:   4, train loss: 1.513157, valid loss: 1.638797, valid accuracy: 43.81%\n",
      "iteration:   1600, epoch:   5, train loss: 1.456494, valid loss: 1.677221, valid accuracy: 41.66%\n",
      "iteration:   1700, epoch:   5, train loss: 1.539097, valid loss: 1.677221, valid accuracy: 41.66%\n",
      "iteration:   1800, epoch:   5, train loss: 1.400892, valid loss: 1.677221, valid accuracy: 41.66%\n",
      "iteration:   1900, epoch:   5, train loss: 1.241925, valid loss: 1.677221, valid accuracy: 41.66%\n",
      "iteration:   2000, epoch:   6, train loss: 1.299767, valid loss: 8.848341, valid accuracy: 32.31%\n",
      "iteration:   2100, epoch:   6, train loss: 1.380590, valid loss: 8.848341, valid accuracy: 32.31%\n",
      "iteration:   2200, epoch:   6, train loss: 1.276041, valid loss: 8.848341, valid accuracy: 32.31%\n",
      "iteration:   2300, epoch:   6, train loss: 1.245555, valid loss: 8.848341, valid accuracy: 32.31%\n",
      "iteration:   2400, epoch:   7, train loss: 0.984103, valid loss: 1.409415, valid accuracy: 51.23%\n",
      "iteration:   2500, epoch:   7, train loss: 1.213108, valid loss: 1.409415, valid accuracy: 51.23%\n",
      "iteration:   2600, epoch:   7, train loss: 1.262861, valid loss: 1.409415, valid accuracy: 51.23%\n",
      "iteration:   2700, epoch:   7, train loss: 1.356809, valid loss: 1.409415, valid accuracy: 51.23%\n",
      "iteration:   2800, epoch:   8, train loss: 1.129917, valid loss: 1.224894, valid accuracy: 64.22%\n",
      "iteration:   2900, epoch:   8, train loss: 1.035509, valid loss: 1.224894, valid accuracy: 64.22%\n",
      "iteration:   3000, epoch:   8, train loss: 0.998187, valid loss: 1.224894, valid accuracy: 64.22%\n",
      "iteration:   3100, epoch:   8, train loss: 0.926480, valid loss: 1.224894, valid accuracy: 64.22%\n",
      "iteration:   3200, epoch:   9, train loss: 1.104629, valid loss: 0.929657, valid accuracy: 71.38%\n",
      "iteration:   3300, epoch:   9, train loss: 1.065247, valid loss: 0.929657, valid accuracy: 71.38%\n",
      "iteration:   3400, epoch:   9, train loss: 0.933124, valid loss: 0.929657, valid accuracy: 71.38%\n",
      "iteration:   3500, epoch:   9, train loss: 0.957853, valid loss: 0.929657, valid accuracy: 71.38%\n",
      "iteration:   3600, epoch:  10, train loss: 1.070413, valid loss: 1.045347, valid accuracy: 67.63%\n",
      "iteration:   3700, epoch:  10, train loss: 0.927974, valid loss: 1.045347, valid accuracy: 67.63%\n",
      "iteration:   3800, epoch:  10, train loss: 1.081241, valid loss: 1.045347, valid accuracy: 67.63%\n",
      "iteration:   3900, epoch:  10, train loss: 0.981037, valid loss: 1.045347, valid accuracy: 67.63%\n",
      "iteration:   4000, epoch:  11, train loss: 0.939989, valid loss: 0.910165, valid accuracy: 69.58%\n",
      "iteration:   4100, epoch:  11, train loss: 1.049583, valid loss: 0.910165, valid accuracy: 69.58%\n",
      "iteration:   4200, epoch:  11, train loss: 0.946662, valid loss: 0.910165, valid accuracy: 69.58%\n",
      "iteration:   4300, epoch:  11, train loss: 0.969335, valid loss: 0.910165, valid accuracy: 69.58%\n",
      "iteration:   4400, epoch:  12, train loss: 1.029602, valid loss: 0.784286, valid accuracy: 73.25%\n",
      "iteration:   4500, epoch:  12, train loss: 0.841792, valid loss: 0.784286, valid accuracy: 73.25%\n",
      "iteration:   4600, epoch:  12, train loss: 1.041285, valid loss: 0.784286, valid accuracy: 73.25%\n",
      "iteration:   4700, epoch:  13, train loss: 0.744415, valid loss: 0.830235, valid accuracy: 73.18%\n",
      "iteration:   4800, epoch:  13, train loss: 0.915697, valid loss: 0.830235, valid accuracy: 73.18%\n",
      "iteration:   4900, epoch:  13, train loss: 0.817792, valid loss: 0.830235, valid accuracy: 73.18%\n",
      "iteration:   5000, epoch:  13, train loss: 1.027559, valid loss: 0.830235, valid accuracy: 73.18%\n",
      "iteration:   5100, epoch:  14, train loss: 0.953901, valid loss: 1.071152, valid accuracy: 66.87%\n",
      "iteration:   5200, epoch:  14, train loss: 0.819837, valid loss: 1.071152, valid accuracy: 66.87%\n",
      "iteration:   5300, epoch:  14, train loss: 0.841876, valid loss: 1.071152, valid accuracy: 66.87%\n",
      "iteration:   5400, epoch:  14, train loss: 0.854256, valid loss: 1.071152, valid accuracy: 66.87%\n",
      "iteration:   5500, epoch:  15, train loss: 0.815339, valid loss: 0.769651, valid accuracy: 74.86%\n",
      "iteration:   5600, epoch:  15, train loss: 0.836567, valid loss: 0.769651, valid accuracy: 74.86%\n",
      "iteration:   5700, epoch:  15, train loss: 0.733546, valid loss: 0.769651, valid accuracy: 74.86%\n",
      "iteration:   5800, epoch:  15, train loss: 0.810185, valid loss: 0.769651, valid accuracy: 74.86%\n",
      "iteration:   5900, epoch:  16, train loss: 0.789519, valid loss: 0.742435, valid accuracy: 75.22%\n",
      "iteration:   6000, epoch:  16, train loss: 0.720111, valid loss: 0.742435, valid accuracy: 75.22%\n",
      "iteration:   6100, epoch:  16, train loss: 0.724867, valid loss: 0.742435, valid accuracy: 75.22%\n",
      "iteration:   6200, epoch:  16, train loss: 0.703871, valid loss: 0.742435, valid accuracy: 75.22%\n",
      "iteration:   6300, epoch:  17, train loss: 0.842814, valid loss: 0.589598, valid accuracy: 80.69%\n",
      "iteration:   6400, epoch:  17, train loss: 0.754092, valid loss: 0.589598, valid accuracy: 80.69%\n",
      "iteration:   6500, epoch:  17, train loss: 0.751783, valid loss: 0.589598, valid accuracy: 80.69%\n",
      "iteration:   6600, epoch:  17, train loss: 0.815925, valid loss: 0.589598, valid accuracy: 80.69%\n",
      "iteration:   6700, epoch:  18, train loss: 0.767909, valid loss: 0.660171, valid accuracy: 78.26%\n",
      "iteration:   6800, epoch:  18, train loss: 0.721436, valid loss: 0.660171, valid accuracy: 78.26%\n",
      "iteration:   6900, epoch:  18, train loss: 0.698662, valid loss: 0.660171, valid accuracy: 78.26%\n",
      "iteration:   7000, epoch:  18, train loss: 0.674524, valid loss: 0.660171, valid accuracy: 78.26%\n",
      "iteration:   7100, epoch:  19, train loss: 0.726623, valid loss: 0.684217, valid accuracy: 76.74%\n",
      "iteration:   7200, epoch:  19, train loss: 0.647660, valid loss: 0.684217, valid accuracy: 76.74%\n",
      "iteration:   7300, epoch:  19, train loss: 0.656602, valid loss: 0.684217, valid accuracy: 76.74%\n",
      "iteration:   7400, epoch:  19, train loss: 0.647706, valid loss: 0.684217, valid accuracy: 76.74%\n",
      "iteration:   7500, epoch:  20, train loss: 0.770426, valid loss: 0.635133, valid accuracy: 78.82%\n",
      "iteration:   7600, epoch:  20, train loss: 0.688053, valid loss: 0.635133, valid accuracy: 78.82%\n",
      "iteration:   7700, epoch:  20, train loss: 0.717532, valid loss: 0.635133, valid accuracy: 78.82%\n",
      "iteration:   7800, epoch:  20, train loss: 0.700700, valid loss: 0.635133, valid accuracy: 78.82%\n",
      "iteration:   7900, epoch:  21, train loss: 0.720529, valid loss: 0.694917, valid accuracy: 77.25%\n",
      "iteration:   8000, epoch:  21, train loss: 0.583274, valid loss: 0.694917, valid accuracy: 77.25%\n",
      "iteration:   8100, epoch:  21, train loss: 0.723196, valid loss: 0.694917, valid accuracy: 77.25%\n",
      "iteration:   8200, epoch:  21, train loss: 0.720262, valid loss: 0.694917, valid accuracy: 77.25%\n",
      "iteration:   8300, epoch:  22, train loss: 0.648787, valid loss: 0.662863, valid accuracy: 78.87%\n",
      "iteration:   8400, epoch:  22, train loss: 0.746070, valid loss: 0.662863, valid accuracy: 78.87%\n",
      "iteration:   8500, epoch:  22, train loss: 0.607124, valid loss: 0.662863, valid accuracy: 78.87%\n",
      "iteration:   8600, epoch:  22, train loss: 0.641244, valid loss: 0.662863, valid accuracy: 78.87%\n",
      "iteration:   8700, epoch:  23, train loss: 0.719296, valid loss: 0.718956, valid accuracy: 76.26%\n",
      "iteration:   8800, epoch:  23, train loss: 0.764437, valid loss: 0.718956, valid accuracy: 76.26%\n",
      "iteration:   8900, epoch:  23, train loss: 0.661630, valid loss: 0.718956, valid accuracy: 76.26%\n",
      "iteration:   9000, epoch:  24, train loss: 0.718521, valid loss: 0.720128, valid accuracy: 76.05%\n",
      "iteration:   9100, epoch:  24, train loss: 0.646258, valid loss: 0.720128, valid accuracy: 76.05%\n",
      "iteration:   9200, epoch:  24, train loss: 0.739867, valid loss: 0.720128, valid accuracy: 76.05%\n",
      "iteration:   9300, epoch:  24, train loss: 0.600195, valid loss: 0.720128, valid accuracy: 76.05%\n",
      "iteration:   9400, epoch:  25, train loss: 0.662471, valid loss: 0.553957, valid accuracy: 81.95%\n",
      "iteration:   9500, epoch:  25, train loss: 0.574309, valid loss: 0.553957, valid accuracy: 81.95%\n",
      "iteration:   9600, epoch:  25, train loss: 0.604469, valid loss: 0.553957, valid accuracy: 81.95%\n",
      "iteration:   9700, epoch:  25, train loss: 0.557056, valid loss: 0.553957, valid accuracy: 81.95%\n",
      "iteration:   9800, epoch:  26, train loss: 0.571542, valid loss: 0.514755, valid accuracy: 84.03%\n",
      "iteration:   9900, epoch:  26, train loss: 0.721211, valid loss: 0.514755, valid accuracy: 84.03%\n",
      "iteration:  10000, epoch:  26, train loss: 0.630570, valid loss: 0.514755, valid accuracy: 84.03%\n",
      "iteration:  10100, epoch:  26, train loss: 0.713993, valid loss: 0.514755, valid accuracy: 84.03%\n",
      "iteration:  10200, epoch:  27, train loss: 0.718501, valid loss: 0.642370, valid accuracy: 78.83%\n",
      "iteration:  10300, epoch:  27, train loss: 0.616322, valid loss: 0.642370, valid accuracy: 78.83%\n",
      "iteration:  10400, epoch:  27, train loss: 0.562383, valid loss: 0.642370, valid accuracy: 78.83%\n",
      "iteration:  10500, epoch:  27, train loss: 0.508110, valid loss: 0.642370, valid accuracy: 78.83%\n",
      "iteration:  10600, epoch:  28, train loss: 0.658319, valid loss: 0.491559, valid accuracy: 83.98%\n",
      "iteration:  10700, epoch:  28, train loss: 0.670647, valid loss: 0.491559, valid accuracy: 83.98%\n",
      "iteration:  10800, epoch:  28, train loss: 0.618427, valid loss: 0.491559, valid accuracy: 83.98%\n",
      "iteration:  10900, epoch:  28, train loss: 0.641953, valid loss: 0.491559, valid accuracy: 83.98%\n",
      "iteration:  11000, epoch:  29, train loss: 0.636071, valid loss: 0.549816, valid accuracy: 81.98%\n",
      "iteration:  11100, epoch:  29, train loss: 0.504086, valid loss: 0.549816, valid accuracy: 81.98%\n",
      "iteration:  11200, epoch:  29, train loss: 0.615990, valid loss: 0.549816, valid accuracy: 81.98%\n",
      "iteration:  11300, epoch:  29, train loss: 0.600741, valid loss: 0.549816, valid accuracy: 81.98%\n",
      "iteration:  11400, epoch:  30, train loss: 0.541623, valid loss: 0.553158, valid accuracy: 81.99%\n",
      "iteration:  11500, epoch:  30, train loss: 0.509083, valid loss: 0.553158, valid accuracy: 81.99%\n",
      "iteration:  11600, epoch:  30, train loss: 0.588296, valid loss: 0.553158, valid accuracy: 81.99%\n",
      "iteration:  11700, epoch:  30, train loss: 0.564898, valid loss: 0.553158, valid accuracy: 81.99%\n",
      "iteration:  11800, epoch:  31, train loss: 0.637485, valid loss: 0.549304, valid accuracy: 82.55%\n",
      "iteration:  11900, epoch:  31, train loss: 0.505716, valid loss: 0.549304, valid accuracy: 82.55%\n",
      "iteration:  12000, epoch:  31, train loss: 0.600046, valid loss: 0.549304, valid accuracy: 82.55%\n",
      "iteration:  12100, epoch:  31, train loss: 0.651083, valid loss: 0.549304, valid accuracy: 82.55%\n",
      "iteration:  12200, epoch:  32, train loss: 0.547548, valid loss: 0.768427, valid accuracy: 75.49%\n",
      "iteration:  12300, epoch:  32, train loss: 0.661490, valid loss: 0.768427, valid accuracy: 75.49%\n",
      "iteration:  12400, epoch:  32, train loss: 0.607058, valid loss: 0.768427, valid accuracy: 75.49%\n",
      "iteration:  12500, epoch:  32, train loss: 0.497890, valid loss: 0.768427, valid accuracy: 75.49%\n",
      "iteration:  12600, epoch:  33, train loss: 0.594005, valid loss: 0.544720, valid accuracy: 82.93%\n",
      "iteration:  12700, epoch:  33, train loss: 0.550788, valid loss: 0.544720, valid accuracy: 82.93%\n",
      "iteration:  12800, epoch:  33, train loss: 0.546978, valid loss: 0.544720, valid accuracy: 82.93%\n",
      "iteration:  12900, epoch:  33, train loss: 0.554256, valid loss: 0.544720, valid accuracy: 82.93%\n",
      "iteration:  13000, epoch:  34, train loss: 0.571037, valid loss: 0.565020, valid accuracy: 82.21%\n",
      "iteration:  13100, epoch:  34, train loss: 0.563115, valid loss: 0.565020, valid accuracy: 82.21%\n",
      "iteration:  13200, epoch:  34, train loss: 0.522561, valid loss: 0.565020, valid accuracy: 82.21%\n",
      "iteration:  13300, epoch:  35, train loss: 0.584247, valid loss: 0.498782, valid accuracy: 84.42%\n",
      "iteration:  13400, epoch:  35, train loss: 0.460087, valid loss: 0.498782, valid accuracy: 84.42%\n",
      "iteration:  13500, epoch:  35, train loss: 0.574508, valid loss: 0.498782, valid accuracy: 84.42%\n",
      "iteration:  13600, epoch:  35, train loss: 0.591672, valid loss: 0.498782, valid accuracy: 84.42%\n",
      "iteration:  13700, epoch:  36, train loss: 0.569841, valid loss: 0.512562, valid accuracy: 83.43%\n",
      "iteration:  13800, epoch:  36, train loss: 0.520714, valid loss: 0.512562, valid accuracy: 83.43%\n",
      "iteration:  13900, epoch:  36, train loss: 0.545479, valid loss: 0.512562, valid accuracy: 83.43%\n",
      "iteration:  14000, epoch:  36, train loss: 0.507474, valid loss: 0.512562, valid accuracy: 83.43%\n",
      "iteration:  14100, epoch:  37, train loss: 0.478034, valid loss: 0.498726, valid accuracy: 84.64%\n",
      "iteration:  14200, epoch:  37, train loss: 0.558854, valid loss: 0.498726, valid accuracy: 84.64%\n",
      "iteration:  14300, epoch:  37, train loss: 0.597382, valid loss: 0.498726, valid accuracy: 84.64%\n",
      "iteration:  14400, epoch:  37, train loss: 0.699893, valid loss: 0.498726, valid accuracy: 84.64%\n",
      "iteration:  14500, epoch:  38, train loss: 0.523606, valid loss: 0.513714, valid accuracy: 83.97%\n",
      "iteration:  14600, epoch:  38, train loss: 0.495086, valid loss: 0.513714, valid accuracy: 83.97%\n",
      "iteration:  14700, epoch:  38, train loss: 0.527846, valid loss: 0.513714, valid accuracy: 83.97%\n",
      "iteration:  14800, epoch:  38, train loss: 0.537037, valid loss: 0.513714, valid accuracy: 83.97%\n",
      "iteration:  14900, epoch:  39, train loss: 0.548708, valid loss: 0.512899, valid accuracy: 83.50%\n",
      "iteration:  15000, epoch:  39, train loss: 0.600995, valid loss: 0.512899, valid accuracy: 83.50%\n",
      "iteration:  15100, epoch:  39, train loss: 0.562142, valid loss: 0.512899, valid accuracy: 83.50%\n",
      "iteration:  15200, epoch:  39, train loss: 0.676118, valid loss: 0.512899, valid accuracy: 83.50%\n",
      "iteration:  15300, epoch:  40, train loss: 0.572196, valid loss: 0.535549, valid accuracy: 82.60%\n",
      "iteration:  15400, epoch:  40, train loss: 0.611741, valid loss: 0.535549, valid accuracy: 82.60%\n",
      "iteration:  15500, epoch:  40, train loss: 0.518936, valid loss: 0.535549, valid accuracy: 82.60%\n",
      "iteration:  15600, epoch:  40, train loss: 0.587758, valid loss: 0.535549, valid accuracy: 82.60%\n",
      "iteration:  15700, epoch:  41, train loss: 0.399852, valid loss: 0.523190, valid accuracy: 83.17%\n",
      "iteration:  15800, epoch:  41, train loss: 0.498521, valid loss: 0.523190, valid accuracy: 83.17%\n",
      "iteration:  15900, epoch:  41, train loss: 0.548785, valid loss: 0.523190, valid accuracy: 83.17%\n",
      "iteration:  16000, epoch:  41, train loss: 0.536980, valid loss: 0.523190, valid accuracy: 83.17%\n",
      "iteration:  16100, epoch:  42, train loss: 0.503172, valid loss: 0.455020, valid accuracy: 85.95%\n",
      "iteration:  16200, epoch:  42, train loss: 0.480496, valid loss: 0.455020, valid accuracy: 85.95%\n",
      "iteration:  16300, epoch:  42, train loss: 0.483267, valid loss: 0.455020, valid accuracy: 85.95%\n",
      "iteration:  16400, epoch:  42, train loss: 0.592150, valid loss: 0.455020, valid accuracy: 85.95%\n",
      "iteration:  16500, epoch:  43, train loss: 0.432404, valid loss: 0.590062, valid accuracy: 81.49%\n",
      "iteration:  16600, epoch:  43, train loss: 0.474484, valid loss: 0.590062, valid accuracy: 81.49%\n",
      "iteration:  16700, epoch:  43, train loss: 0.586273, valid loss: 0.590062, valid accuracy: 81.49%\n",
      "iteration:  16800, epoch:  43, train loss: 0.540652, valid loss: 0.590062, valid accuracy: 81.49%\n",
      "iteration:  16900, epoch:  44, train loss: 0.457198, valid loss: 0.632149, valid accuracy: 80.30%\n",
      "iteration:  17000, epoch:  44, train loss: 0.439159, valid loss: 0.632149, valid accuracy: 80.30%\n",
      "iteration:  17100, epoch:  44, train loss: 0.601073, valid loss: 0.632149, valid accuracy: 80.30%\n",
      "iteration:  17200, epoch:  44, train loss: 0.458786, valid loss: 0.632149, valid accuracy: 80.30%\n",
      "iteration:  17300, epoch:  45, train loss: 0.567590, valid loss: 0.463148, valid accuracy: 85.40%\n",
      "iteration:  17400, epoch:  45, train loss: 0.540333, valid loss: 0.463148, valid accuracy: 85.40%\n",
      "iteration:  17500, epoch:  45, train loss: 0.612832, valid loss: 0.463148, valid accuracy: 85.40%\n",
      "iteration:  17600, epoch:  46, train loss: 0.538180, valid loss: 0.427462, valid accuracy: 86.65%\n",
      "iteration:  17700, epoch:  46, train loss: 0.541704, valid loss: 0.427462, valid accuracy: 86.65%\n",
      "iteration:  17800, epoch:  46, train loss: 0.458625, valid loss: 0.427462, valid accuracy: 86.65%\n",
      "iteration:  17900, epoch:  46, train loss: 0.553773, valid loss: 0.427462, valid accuracy: 86.65%\n",
      "iteration:  18000, epoch:  47, train loss: 0.455446, valid loss: 0.456016, valid accuracy: 85.91%\n",
      "iteration:  18100, epoch:  47, train loss: 0.553676, valid loss: 0.456016, valid accuracy: 85.91%\n",
      "iteration:  18200, epoch:  47, train loss: 0.502936, valid loss: 0.456016, valid accuracy: 85.91%\n",
      "iteration:  18300, epoch:  47, train loss: 0.538361, valid loss: 0.456016, valid accuracy: 85.91%\n",
      "iteration:  18400, epoch:  48, train loss: 0.468346, valid loss: 0.506170, valid accuracy: 84.39%\n",
      "iteration:  18500, epoch:  48, train loss: 0.514596, valid loss: 0.506170, valid accuracy: 84.39%\n",
      "iteration:  18600, epoch:  48, train loss: 0.625048, valid loss: 0.506170, valid accuracy: 84.39%\n",
      "iteration:  18700, epoch:  48, train loss: 0.504059, valid loss: 0.506170, valid accuracy: 84.39%\n",
      "iteration:  18800, epoch:  49, train loss: 0.459604, valid loss: 0.490108, valid accuracy: 84.93%\n",
      "iteration:  18900, epoch:  49, train loss: 0.497275, valid loss: 0.490108, valid accuracy: 84.93%\n",
      "iteration:  19000, epoch:  49, train loss: 0.466318, valid loss: 0.490108, valid accuracy: 84.93%\n",
      "iteration:  19100, epoch:  49, train loss: 0.447212, valid loss: 0.490108, valid accuracy: 84.93%\n",
      "iteration:  19200, epoch:  50, train loss: 0.462914, valid loss: 0.570566, valid accuracy: 82.40%\n",
      "iteration:  19300, epoch:  50, train loss: 0.500162, valid loss: 0.570566, valid accuracy: 82.40%\n",
      "iteration:  19400, epoch:  50, train loss: 0.578571, valid loss: 0.570566, valid accuracy: 82.40%\n",
      "iteration:  19500, epoch:  50, train loss: 0.509316, valid loss: 0.570566, valid accuracy: 82.40%\n",
      "iteration:  19600, epoch:  51, train loss: 0.526519, valid loss: 0.504023, valid accuracy: 84.65%\n",
      "iteration:  19700, epoch:  51, train loss: 0.549977, valid loss: 0.504023, valid accuracy: 84.65%\n",
      "iteration:  19800, epoch:  51, train loss: 0.582467, valid loss: 0.504023, valid accuracy: 84.65%\n",
      "iteration:  19900, epoch:  51, train loss: 0.542893, valid loss: 0.504023, valid accuracy: 84.65%\n",
      "iteration:  20000, epoch:  52, train loss: 0.460520, valid loss: 0.749470, valid accuracy: 77.58%\n",
      "iteration:  20100, epoch:  52, train loss: 0.414072, valid loss: 0.749470, valid accuracy: 77.58%\n",
      "iteration:  20200, epoch:  52, train loss: 0.512661, valid loss: 0.749470, valid accuracy: 77.58%\n",
      "iteration:  20300, epoch:  52, train loss: 0.456073, valid loss: 0.749470, valid accuracy: 77.58%\n",
      "iteration:  20400, epoch:  53, train loss: 0.456511, valid loss: 0.451849, valid accuracy: 86.09%\n",
      "iteration:  20500, epoch:  53, train loss: 0.423644, valid loss: 0.451849, valid accuracy: 86.09%\n",
      "iteration:  20600, epoch:  53, train loss: 0.400449, valid loss: 0.451849, valid accuracy: 86.09%\n",
      "iteration:  20700, epoch:  53, train loss: 0.515448, valid loss: 0.451849, valid accuracy: 86.09%\n",
      "iteration:  20800, epoch:  54, train loss: 0.488993, valid loss: 0.486095, valid accuracy: 85.05%\n",
      "iteration:  20900, epoch:  54, train loss: 0.507261, valid loss: 0.486095, valid accuracy: 85.05%\n",
      "iteration:  21000, epoch:  54, train loss: 0.539235, valid loss: 0.486095, valid accuracy: 85.05%\n",
      "iteration:  21100, epoch:  54, train loss: 0.461312, valid loss: 0.486095, valid accuracy: 85.05%\n",
      "iteration:  21200, epoch:  55, train loss: 0.442851, valid loss: 0.542863, valid accuracy: 83.33%\n",
      "iteration:  21300, epoch:  55, train loss: 0.472989, valid loss: 0.542863, valid accuracy: 83.33%\n",
      "iteration:  21400, epoch:  55, train loss: 0.501031, valid loss: 0.542863, valid accuracy: 83.33%\n",
      "iteration:  21500, epoch:  55, train loss: 0.476022, valid loss: 0.542863, valid accuracy: 83.33%\n",
      "iteration:  21600, epoch:  56, train loss: 0.500654, valid loss: 0.447469, valid accuracy: 86.14%\n",
      "iteration:  21700, epoch:  56, train loss: 0.485862, valid loss: 0.447469, valid accuracy: 86.14%\n",
      "iteration:  21800, epoch:  56, train loss: 0.525481, valid loss: 0.447469, valid accuracy: 86.14%\n",
      "iteration:  21900, epoch:  57, train loss: 0.467352, valid loss: 0.445859, valid accuracy: 86.28%\n",
      "iteration:  22000, epoch:  57, train loss: 0.415983, valid loss: 0.445859, valid accuracy: 86.28%\n",
      "iteration:  22100, epoch:  57, train loss: 0.513425, valid loss: 0.445859, valid accuracy: 86.28%\n",
      "iteration:  22200, epoch:  57, train loss: 0.465087, valid loss: 0.445859, valid accuracy: 86.28%\n",
      "iteration:  22300, epoch:  58, train loss: 0.434925, valid loss: 0.519077, valid accuracy: 84.35%\n",
      "iteration:  22400, epoch:  58, train loss: 0.451338, valid loss: 0.519077, valid accuracy: 84.35%\n",
      "iteration:  22500, epoch:  58, train loss: 0.498564, valid loss: 0.519077, valid accuracy: 84.35%\n",
      "iteration:  22600, epoch:  58, train loss: 0.420209, valid loss: 0.519077, valid accuracy: 84.35%\n",
      "iteration:  22700, epoch:  59, train loss: 0.404183, valid loss: 0.377645, valid accuracy: 88.01%\n",
      "iteration:  22800, epoch:  59, train loss: 0.460703, valid loss: 0.377645, valid accuracy: 88.01%\n",
      "iteration:  22900, epoch:  59, train loss: 0.382730, valid loss: 0.377645, valid accuracy: 88.01%\n",
      "iteration:  23000, epoch:  59, train loss: 0.471333, valid loss: 0.377645, valid accuracy: 88.01%\n",
      "iteration:  23100, epoch:  60, train loss: 0.440860, valid loss: 0.464895, valid accuracy: 85.74%\n",
      "iteration:  23200, epoch:  60, train loss: 0.421800, valid loss: 0.464895, valid accuracy: 85.74%\n",
      "iteration:  23300, epoch:  60, train loss: 0.452237, valid loss: 0.464895, valid accuracy: 85.74%\n",
      "iteration:  23400, epoch:  60, train loss: 0.526184, valid loss: 0.464895, valid accuracy: 85.74%\n",
      "iteration:  23500, epoch:  61, train loss: 0.407334, valid loss: 0.495919, valid accuracy: 85.15%\n",
      "iteration:  23600, epoch:  61, train loss: 0.566732, valid loss: 0.495919, valid accuracy: 85.15%\n",
      "iteration:  23700, epoch:  61, train loss: 0.484566, valid loss: 0.495919, valid accuracy: 85.15%\n",
      "iteration:  23800, epoch:  61, train loss: 0.493023, valid loss: 0.495919, valid accuracy: 85.15%\n",
      "iteration:  23900, epoch:  62, train loss: 0.471922, valid loss: 0.379462, valid accuracy: 88.20%\n",
      "iteration:  24000, epoch:  62, train loss: 0.521075, valid loss: 0.379462, valid accuracy: 88.20%\n",
      "iteration:  24100, epoch:  62, train loss: 0.444205, valid loss: 0.379462, valid accuracy: 88.20%\n",
      "iteration:  24200, epoch:  62, train loss: 0.471422, valid loss: 0.379462, valid accuracy: 88.20%\n",
      "iteration:  24300, epoch:  63, train loss: 0.589818, valid loss: 0.380818, valid accuracy: 88.46%\n",
      "iteration:  24400, epoch:  63, train loss: 0.473558, valid loss: 0.380818, valid accuracy: 88.46%\n",
      "iteration:  24500, epoch:  63, train loss: 0.465478, valid loss: 0.380818, valid accuracy: 88.46%\n",
      "iteration:  24600, epoch:  63, train loss: 0.519132, valid loss: 0.380818, valid accuracy: 88.46%\n",
      "iteration:  24700, epoch:  64, train loss: 0.499252, valid loss: 0.594707, valid accuracy: 82.15%\n",
      "iteration:  24800, epoch:  64, train loss: 0.445067, valid loss: 0.594707, valid accuracy: 82.15%\n",
      "iteration:  24900, epoch:  64, train loss: 0.423762, valid loss: 0.594707, valid accuracy: 82.15%\n",
      "iteration:  25000, epoch:  64, train loss: 0.400997, valid loss: 0.594707, valid accuracy: 82.15%\n",
      "iteration:  25100, epoch:  65, train loss: 0.389152, valid loss: 0.363387, valid accuracy: 89.39%\n",
      "iteration:  25200, epoch:  65, train loss: 0.500551, valid loss: 0.363387, valid accuracy: 89.39%\n",
      "iteration:  25300, epoch:  65, train loss: 0.467402, valid loss: 0.363387, valid accuracy: 89.39%\n",
      "iteration:  25400, epoch:  65, train loss: 0.507910, valid loss: 0.363387, valid accuracy: 89.39%\n",
      "iteration:  25500, epoch:  66, train loss: 0.457133, valid loss: 0.455281, valid accuracy: 86.37%\n",
      "iteration:  25600, epoch:  66, train loss: 0.455668, valid loss: 0.455281, valid accuracy: 86.37%\n",
      "iteration:  25700, epoch:  66, train loss: 0.496223, valid loss: 0.455281, valid accuracy: 86.37%\n",
      "iteration:  25800, epoch:  66, train loss: 0.402532, valid loss: 0.455281, valid accuracy: 86.37%\n",
      "iteration:  25900, epoch:  67, train loss: 0.460592, valid loss: 0.456377, valid accuracy: 86.72%\n",
      "iteration:  26000, epoch:  67, train loss: 0.463223, valid loss: 0.456377, valid accuracy: 86.72%\n",
      "iteration:  26100, epoch:  67, train loss: 0.472218, valid loss: 0.456377, valid accuracy: 86.72%\n",
      "iteration:  26200, epoch:  68, train loss: 0.418062, valid loss: 0.452922, valid accuracy: 86.48%\n",
      "iteration:  26300, epoch:  68, train loss: 0.510222, valid loss: 0.452922, valid accuracy: 86.48%\n",
      "iteration:  26400, epoch:  68, train loss: 0.383249, valid loss: 0.452922, valid accuracy: 86.48%\n",
      "iteration:  26500, epoch:  68, train loss: 0.379040, valid loss: 0.452922, valid accuracy: 86.48%\n",
      "iteration:  26600, epoch:  69, train loss: 0.481420, valid loss: 0.367433, valid accuracy: 88.83%\n",
      "iteration:  26700, epoch:  69, train loss: 0.369975, valid loss: 0.367433, valid accuracy: 88.83%\n",
      "iteration:  26800, epoch:  69, train loss: 0.426604, valid loss: 0.367433, valid accuracy: 88.83%\n",
      "iteration:  26900, epoch:  69, train loss: 0.389183, valid loss: 0.367433, valid accuracy: 88.83%\n",
      "iteration:  27000, epoch:  70, train loss: 0.465764, valid loss: 0.463033, valid accuracy: 86.27%\n",
      "iteration:  27100, epoch:  70, train loss: 0.592074, valid loss: 0.463033, valid accuracy: 86.27%\n",
      "iteration:  27200, epoch:  70, train loss: 0.452682, valid loss: 0.463033, valid accuracy: 86.27%\n",
      "iteration:  27300, epoch:  70, train loss: 0.522963, valid loss: 0.463033, valid accuracy: 86.27%\n",
      "iteration:  27400, epoch:  71, train loss: 0.387012, valid loss: 0.356894, valid accuracy: 89.03%\n",
      "iteration:  27500, epoch:  71, train loss: 0.453953, valid loss: 0.356894, valid accuracy: 89.03%\n",
      "iteration:  27600, epoch:  71, train loss: 0.484219, valid loss: 0.356894, valid accuracy: 89.03%\n",
      "iteration:  27700, epoch:  71, train loss: 0.450648, valid loss: 0.356894, valid accuracy: 89.03%\n",
      "iteration:  27800, epoch:  72, train loss: 0.478255, valid loss: 0.546287, valid accuracy: 83.86%\n",
      "iteration:  27900, epoch:  72, train loss: 0.517110, valid loss: 0.546287, valid accuracy: 83.86%\n",
      "iteration:  28000, epoch:  72, train loss: 0.434822, valid loss: 0.546287, valid accuracy: 83.86%\n",
      "iteration:  28100, epoch:  72, train loss: 0.384092, valid loss: 0.546287, valid accuracy: 83.86%\n",
      "iteration:  28200, epoch:  73, train loss: 0.398625, valid loss: 0.462116, valid accuracy: 86.04%\n",
      "iteration:  28300, epoch:  73, train loss: 0.467657, valid loss: 0.462116, valid accuracy: 86.04%\n",
      "iteration:  28400, epoch:  73, train loss: 0.430709, valid loss: 0.462116, valid accuracy: 86.04%\n",
      "iteration:  28500, epoch:  73, train loss: 0.451791, valid loss: 0.462116, valid accuracy: 86.04%\n",
      "iteration:  28600, epoch:  74, train loss: 0.491081, valid loss: 0.453559, valid accuracy: 85.96%\n",
      "iteration:  28700, epoch:  74, train loss: 0.407628, valid loss: 0.453559, valid accuracy: 85.96%\n",
      "iteration:  28800, epoch:  74, train loss: 0.509368, valid loss: 0.453559, valid accuracy: 85.96%\n",
      "iteration:  28900, epoch:  74, train loss: 0.475915, valid loss: 0.453559, valid accuracy: 85.96%\n",
      "iteration:  29000, epoch:  75, train loss: 0.423286, valid loss: 0.376641, valid accuracy: 88.98%\n",
      "iteration:  29100, epoch:  75, train loss: 0.553346, valid loss: 0.376641, valid accuracy: 88.98%\n",
      "iteration:  29200, epoch:  75, train loss: 0.500479, valid loss: 0.376641, valid accuracy: 88.98%\n",
      "iteration:  29300, epoch:  75, train loss: 0.465128, valid loss: 0.376641, valid accuracy: 88.98%\n",
      "iteration:  29400, epoch:  76, train loss: 0.478239, valid loss: 0.409373, valid accuracy: 87.34%\n",
      "iteration:  29500, epoch:  76, train loss: 0.455122, valid loss: 0.409373, valid accuracy: 87.34%\n",
      "iteration:  29600, epoch:  76, train loss: 0.413989, valid loss: 0.409373, valid accuracy: 87.34%\n",
      "iteration:  29700, epoch:  76, train loss: 0.417193, valid loss: 0.409373, valid accuracy: 87.34%\n",
      "iteration:  29800, epoch:  77, train loss: 0.436737, valid loss: 0.597613, valid accuracy: 82.69%\n",
      "iteration:  29900, epoch:  77, train loss: 0.403136, valid loss: 0.597613, valid accuracy: 82.69%\n",
      "iteration:  30000, epoch:  77, train loss: 0.489438, valid loss: 0.597613, valid accuracy: 82.69%\n",
      "iteration:  30100, epoch:  77, train loss: 0.421579, valid loss: 0.597613, valid accuracy: 82.69%\n",
      "iteration:  30200, epoch:  78, train loss: 0.452187, valid loss: 0.495096, valid accuracy: 84.76%\n",
      "iteration:  30300, epoch:  78, train loss: 0.370296, valid loss: 0.495096, valid accuracy: 84.76%\n",
      "iteration:  30400, epoch:  78, train loss: 0.432129, valid loss: 0.495096, valid accuracy: 84.76%\n",
      "iteration:  30500, epoch:  79, train loss: 0.458948, valid loss: 0.441948, valid accuracy: 86.88%\n",
      "iteration:  30600, epoch:  79, train loss: 0.430360, valid loss: 0.441948, valid accuracy: 86.88%\n",
      "iteration:  30700, epoch:  79, train loss: 0.430863, valid loss: 0.441948, valid accuracy: 86.88%\n",
      "iteration:  30800, epoch:  79, train loss: 0.437778, valid loss: 0.441948, valid accuracy: 86.88%\n",
      "iteration:  30900, epoch:  80, train loss: 0.342204, valid loss: 0.369470, valid accuracy: 88.59%\n",
      "iteration:  31000, epoch:  80, train loss: 0.428126, valid loss: 0.369470, valid accuracy: 88.59%\n",
      "iteration:  31100, epoch:  80, train loss: 0.491212, valid loss: 0.369470, valid accuracy: 88.59%\n",
      "iteration:  31200, epoch:  80, train loss: 0.454161, valid loss: 0.369470, valid accuracy: 88.59%\n",
      "iteration:  31300, epoch:  81, train loss: 0.464170, valid loss: 0.497362, valid accuracy: 85.53%\n",
      "iteration:  31400, epoch:  81, train loss: 0.362239, valid loss: 0.497362, valid accuracy: 85.53%\n",
      "iteration:  31500, epoch:  81, train loss: 0.442584, valid loss: 0.497362, valid accuracy: 85.53%\n",
      "iteration:  31600, epoch:  81, train loss: 0.439325, valid loss: 0.497362, valid accuracy: 85.53%\n",
      "iteration:  31700, epoch:  82, train loss: 0.456623, valid loss: 0.489194, valid accuracy: 85.88%\n",
      "iteration:  31800, epoch:  82, train loss: 0.384087, valid loss: 0.489194, valid accuracy: 85.88%\n",
      "iteration:  31900, epoch:  82, train loss: 0.427529, valid loss: 0.489194, valid accuracy: 85.88%\n",
      "iteration:  32000, epoch:  82, train loss: 0.493375, valid loss: 0.489194, valid accuracy: 85.88%\n",
      "iteration:  32100, epoch:  83, train loss: 0.537498, valid loss: 0.438610, valid accuracy: 86.97%\n",
      "iteration:  32200, epoch:  83, train loss: 0.409493, valid loss: 0.438610, valid accuracy: 86.97%\n",
      "iteration:  32300, epoch:  83, train loss: 0.402640, valid loss: 0.438610, valid accuracy: 86.97%\n",
      "iteration:  32400, epoch:  83, train loss: 0.404668, valid loss: 0.438610, valid accuracy: 86.97%\n",
      "iteration:  32500, epoch:  84, train loss: 0.407180, valid loss: 0.390547, valid accuracy: 88.40%\n",
      "iteration:  32600, epoch:  84, train loss: 0.429030, valid loss: 0.390547, valid accuracy: 88.40%\n",
      "iteration:  32700, epoch:  84, train loss: 0.365548, valid loss: 0.390547, valid accuracy: 88.40%\n",
      "iteration:  32800, epoch:  84, train loss: 0.366250, valid loss: 0.390547, valid accuracy: 88.40%\n",
      "iteration:  32900, epoch:  85, train loss: 0.338950, valid loss: 0.440752, valid accuracy: 87.20%\n",
      "iteration:  33000, epoch:  85, train loss: 0.398072, valid loss: 0.440752, valid accuracy: 87.20%\n",
      "iteration:  33100, epoch:  85, train loss: 0.432003, valid loss: 0.440752, valid accuracy: 87.20%\n",
      "iteration:  33200, epoch:  85, train loss: 0.401945, valid loss: 0.440752, valid accuracy: 87.20%\n",
      "iteration:  33300, epoch:  86, train loss: 0.377702, valid loss: 0.402452, valid accuracy: 88.41%\n",
      "iteration:  33400, epoch:  86, train loss: 0.430363, valid loss: 0.402452, valid accuracy: 88.41%\n",
      "iteration:  33500, epoch:  86, train loss: 0.457542, valid loss: 0.402452, valid accuracy: 88.41%\n",
      "iteration:  33600, epoch:  86, train loss: 0.431940, valid loss: 0.402452, valid accuracy: 88.41%\n",
      "iteration:  33700, epoch:  87, train loss: 0.457040, valid loss: 0.380026, valid accuracy: 88.61%\n",
      "iteration:  33800, epoch:  87, train loss: 0.399935, valid loss: 0.380026, valid accuracy: 88.61%\n",
      "iteration:  33900, epoch:  87, train loss: 0.463818, valid loss: 0.380026, valid accuracy: 88.61%\n",
      "iteration:  34000, epoch:  87, train loss: 0.348496, valid loss: 0.380026, valid accuracy: 88.61%\n",
      "iteration:  34100, epoch:  88, train loss: 0.412368, valid loss: 0.420115, valid accuracy: 87.40%\n",
      "iteration:  34200, epoch:  88, train loss: 0.430522, valid loss: 0.420115, valid accuracy: 87.40%\n",
      "iteration:  34300, epoch:  88, train loss: 0.442856, valid loss: 0.420115, valid accuracy: 87.40%\n",
      "iteration:  34400, epoch:  88, train loss: 0.410248, valid loss: 0.420115, valid accuracy: 87.40%\n",
      "iteration:  34500, epoch:  89, train loss: 0.444570, valid loss: 0.474325, valid accuracy: 86.21%\n",
      "iteration:  34600, epoch:  89, train loss: 0.416420, valid loss: 0.474325, valid accuracy: 86.21%\n",
      "iteration:  34700, epoch:  89, train loss: 0.502469, valid loss: 0.474325, valid accuracy: 86.21%\n",
      "iteration:  34800, epoch:  90, train loss: 0.445107, valid loss: 0.365305, valid accuracy: 89.15%\n",
      "iteration:  34900, epoch:  90, train loss: 0.371290, valid loss: 0.365305, valid accuracy: 89.15%\n",
      "iteration:  35000, epoch:  90, train loss: 0.433245, valid loss: 0.365305, valid accuracy: 89.15%\n",
      "iteration:  35100, epoch:  90, train loss: 0.403718, valid loss: 0.365305, valid accuracy: 89.15%\n",
      "iteration:  35200, epoch:  91, train loss: 0.398704, valid loss: 0.481822, valid accuracy: 85.68%\n",
      "iteration:  35300, epoch:  91, train loss: 0.433806, valid loss: 0.481822, valid accuracy: 85.68%\n",
      "iteration:  35400, epoch:  91, train loss: 0.422523, valid loss: 0.481822, valid accuracy: 85.68%\n",
      "iteration:  35500, epoch:  91, train loss: 0.471270, valid loss: 0.481822, valid accuracy: 85.68%\n",
      "iteration:  35600, epoch:  92, train loss: 0.396288, valid loss: 0.493576, valid accuracy: 85.27%\n",
      "iteration:  35700, epoch:  92, train loss: 0.529984, valid loss: 0.493576, valid accuracy: 85.27%\n",
      "iteration:  35800, epoch:  92, train loss: 0.410633, valid loss: 0.493576, valid accuracy: 85.27%\n",
      "iteration:  35900, epoch:  92, train loss: 0.420660, valid loss: 0.493576, valid accuracy: 85.27%\n",
      "iteration:  36000, epoch:  93, train loss: 0.391648, valid loss: 0.412360, valid accuracy: 87.97%\n",
      "iteration:  36100, epoch:  93, train loss: 0.444975, valid loss: 0.412360, valid accuracy: 87.97%\n",
      "iteration:  36200, epoch:  93, train loss: 0.381229, valid loss: 0.412360, valid accuracy: 87.97%\n",
      "iteration:  36300, epoch:  93, train loss: 0.415450, valid loss: 0.412360, valid accuracy: 87.97%\n",
      "iteration:  36400, epoch:  94, train loss: 0.406816, valid loss: 0.461145, valid accuracy: 87.08%\n",
      "iteration:  36500, epoch:  94, train loss: 0.393873, valid loss: 0.461145, valid accuracy: 87.08%\n",
      "iteration:  36600, epoch:  94, train loss: 0.441451, valid loss: 0.461145, valid accuracy: 87.08%\n",
      "iteration:  36700, epoch:  94, train loss: 0.423583, valid loss: 0.461145, valid accuracy: 87.08%\n",
      "iteration:  36800, epoch:  95, train loss: 0.401181, valid loss: 0.381825, valid accuracy: 88.52%\n",
      "iteration:  36900, epoch:  95, train loss: 0.478603, valid loss: 0.381825, valid accuracy: 88.52%\n",
      "iteration:  37000, epoch:  95, train loss: 0.382256, valid loss: 0.381825, valid accuracy: 88.52%\n",
      "iteration:  37100, epoch:  95, train loss: 0.375160, valid loss: 0.381825, valid accuracy: 88.52%\n",
      "iteration:  37200, epoch:  96, train loss: 0.397849, valid loss: 0.455504, valid accuracy: 86.95%\n",
      "iteration:  37300, epoch:  96, train loss: 0.408399, valid loss: 0.455504, valid accuracy: 86.95%\n",
      "iteration:  37400, epoch:  96, train loss: 0.436218, valid loss: 0.455504, valid accuracy: 86.95%\n",
      "iteration:  37500, epoch:  96, train loss: 0.386415, valid loss: 0.455504, valid accuracy: 86.95%\n",
      "iteration:  37600, epoch:  97, train loss: 0.438472, valid loss: 0.393762, valid accuracy: 88.04%\n",
      "iteration:  37700, epoch:  97, train loss: 0.421574, valid loss: 0.393762, valid accuracy: 88.04%\n",
      "iteration:  37800, epoch:  97, train loss: 0.387442, valid loss: 0.393762, valid accuracy: 88.04%\n",
      "iteration:  37900, epoch:  97, train loss: 0.436906, valid loss: 0.393762, valid accuracy: 88.04%\n",
      "iteration:  38000, epoch:  98, train loss: 0.470916, valid loss: 0.404982, valid accuracy: 87.95%\n",
      "iteration:  38100, epoch:  98, train loss: 0.526034, valid loss: 0.404982, valid accuracy: 87.95%\n",
      "iteration:  38200, epoch:  98, train loss: 0.360011, valid loss: 0.404982, valid accuracy: 87.95%\n",
      "iteration:  38300, epoch:  98, train loss: 0.405450, valid loss: 0.404982, valid accuracy: 87.95%\n",
      "iteration:  38400, epoch:  99, train loss: 0.389771, valid loss: 0.432286, valid accuracy: 87.77%\n",
      "iteration:  38500, epoch:  99, train loss: 0.378703, valid loss: 0.432286, valid accuracy: 87.77%\n",
      "iteration:  38600, epoch:  99, train loss: 0.422440, valid loss: 0.432286, valid accuracy: 87.77%\n",
      "iteration:  38700, epoch:  99, train loss: 0.448531, valid loss: 0.432286, valid accuracy: 87.77%\n",
      "iteration:  38800, epoch: 100, train loss: 0.450110, valid loss: 0.359079, valid accuracy: 89.56%\n",
      "iteration:  38900, epoch: 100, train loss: 0.419856, valid loss: 0.359079, valid accuracy: 89.56%\n",
      "iteration:  39000, epoch: 100, train loss: 0.463974, valid loss: 0.359079, valid accuracy: 89.56%\n",
      "iteration:  39100, epoch: 100, train loss: 0.333169, valid loss: 0.359079, valid accuracy: 89.56%\n",
      "iteration:  39200, epoch: 101, train loss: 0.369619, valid loss: 0.396074, valid accuracy: 88.38%\n",
      "iteration:  39300, epoch: 101, train loss: 0.330469, valid loss: 0.396074, valid accuracy: 88.38%\n",
      "iteration:  39400, epoch: 101, train loss: 0.473873, valid loss: 0.396074, valid accuracy: 88.38%\n",
      "iteration:  39500, epoch: 102, train loss: 0.348208, valid loss: 0.413922, valid accuracy: 87.76%\n",
      "iteration:  39600, epoch: 102, train loss: 0.441176, valid loss: 0.413922, valid accuracy: 87.76%\n",
      "iteration:  39700, epoch: 102, train loss: 0.386167, valid loss: 0.413922, valid accuracy: 87.76%\n",
      "iteration:  39800, epoch: 102, train loss: 0.373487, valid loss: 0.413922, valid accuracy: 87.76%\n",
      "iteration:  39900, epoch: 103, train loss: 0.390520, valid loss: 0.427445, valid accuracy: 87.49%\n",
      "iteration:  40000, epoch: 103, train loss: 0.404394, valid loss: 0.427445, valid accuracy: 87.49%\n",
      "iteration:  40100, epoch: 103, train loss: 0.375932, valid loss: 0.427445, valid accuracy: 87.49%\n",
      "iteration:  40200, epoch: 103, train loss: 0.427980, valid loss: 0.427445, valid accuracy: 87.49%\n",
      "iteration:  40300, epoch: 104, train loss: 0.367326, valid loss: 0.446900, valid accuracy: 87.20%\n",
      "iteration:  40400, epoch: 104, train loss: 0.366173, valid loss: 0.446900, valid accuracy: 87.20%\n",
      "iteration:  40500, epoch: 104, train loss: 0.424249, valid loss: 0.446900, valid accuracy: 87.20%\n",
      "iteration:  40600, epoch: 104, train loss: 0.387706, valid loss: 0.446900, valid accuracy: 87.20%\n",
      "iteration:  40700, epoch: 105, train loss: 0.436888, valid loss: 0.407170, valid accuracy: 88.14%\n",
      "iteration:  40800, epoch: 105, train loss: 0.382816, valid loss: 0.407170, valid accuracy: 88.14%\n",
      "iteration:  40900, epoch: 105, train loss: 0.398069, valid loss: 0.407170, valid accuracy: 88.14%\n",
      "iteration:  41000, epoch: 105, train loss: 0.449577, valid loss: 0.407170, valid accuracy: 88.14%\n",
      "iteration:  41100, epoch: 106, train loss: 0.455756, valid loss: 0.349476, valid accuracy: 89.70%\n",
      "iteration:  41200, epoch: 106, train loss: 0.356151, valid loss: 0.349476, valid accuracy: 89.70%\n",
      "iteration:  41300, epoch: 106, train loss: 0.348776, valid loss: 0.349476, valid accuracy: 89.70%\n",
      "iteration:  41400, epoch: 106, train loss: 0.378852, valid loss: 0.349476, valid accuracy: 89.70%\n",
      "iteration:  41500, epoch: 107, train loss: 0.359878, valid loss: 0.396280, valid accuracy: 88.31%\n",
      "iteration:  41600, epoch: 107, train loss: 0.434005, valid loss: 0.396280, valid accuracy: 88.31%\n",
      "iteration:  41700, epoch: 107, train loss: 0.393955, valid loss: 0.396280, valid accuracy: 88.31%\n",
      "iteration:  41800, epoch: 107, train loss: 0.482056, valid loss: 0.396280, valid accuracy: 88.31%\n",
      "iteration:  41900, epoch: 108, train loss: 0.429829, valid loss: 0.393132, valid accuracy: 88.79%\n",
      "iteration:  42000, epoch: 108, train loss: 0.374758, valid loss: 0.393132, valid accuracy: 88.79%\n",
      "iteration:  42100, epoch: 108, train loss: 0.434640, valid loss: 0.393132, valid accuracy: 88.79%\n",
      "iteration:  42200, epoch: 108, train loss: 0.436613, valid loss: 0.393132, valid accuracy: 88.79%\n",
      "iteration:  42300, epoch: 109, train loss: 0.413494, valid loss: 0.397938, valid accuracy: 88.20%\n",
      "iteration:  42400, epoch: 109, train loss: 0.386647, valid loss: 0.397938, valid accuracy: 88.20%\n",
      "iteration:  42500, epoch: 109, train loss: 0.363089, valid loss: 0.397938, valid accuracy: 88.20%\n",
      "iteration:  42600, epoch: 109, train loss: 0.375363, valid loss: 0.397938, valid accuracy: 88.20%\n",
      "iteration:  42700, epoch: 110, train loss: 0.395005, valid loss: 0.409485, valid accuracy: 88.22%\n",
      "iteration:  42800, epoch: 110, train loss: 0.384555, valid loss: 0.409485, valid accuracy: 88.22%\n",
      "iteration:  42900, epoch: 110, train loss: 0.413626, valid loss: 0.409485, valid accuracy: 88.22%\n",
      "iteration:  43000, epoch: 110, train loss: 0.372842, valid loss: 0.409485, valid accuracy: 88.22%\n",
      "iteration:  43100, epoch: 111, train loss: 0.419480, valid loss: 0.360452, valid accuracy: 89.49%\n",
      "iteration:  43200, epoch: 111, train loss: 0.426580, valid loss: 0.360452, valid accuracy: 89.49%\n",
      "iteration:  43300, epoch: 111, train loss: 0.351745, valid loss: 0.360452, valid accuracy: 89.49%\n",
      "iteration:  43400, epoch: 111, train loss: 0.370370, valid loss: 0.360452, valid accuracy: 89.49%\n",
      "iteration:  43500, epoch: 112, train loss: 0.355274, valid loss: 0.360394, valid accuracy: 89.39%\n",
      "iteration:  43600, epoch: 112, train loss: 0.413752, valid loss: 0.360394, valid accuracy: 89.39%\n",
      "iteration:  43700, epoch: 112, train loss: 0.414132, valid loss: 0.360394, valid accuracy: 89.39%\n",
      "iteration:  43800, epoch: 113, train loss: 0.350715, valid loss: 0.336815, valid accuracy: 90.36%\n",
      "iteration:  43900, epoch: 113, train loss: 0.371321, valid loss: 0.336815, valid accuracy: 90.36%\n",
      "iteration:  44000, epoch: 113, train loss: 0.364670, valid loss: 0.336815, valid accuracy: 90.36%\n",
      "iteration:  44100, epoch: 113, train loss: 0.427167, valid loss: 0.336815, valid accuracy: 90.36%\n",
      "iteration:  44200, epoch: 114, train loss: 0.416781, valid loss: 0.322107, valid accuracy: 90.76%\n",
      "iteration:  44300, epoch: 114, train loss: 0.428980, valid loss: 0.322107, valid accuracy: 90.76%\n",
      "iteration:  44400, epoch: 114, train loss: 0.386294, valid loss: 0.322107, valid accuracy: 90.76%\n",
      "iteration:  44500, epoch: 114, train loss: 0.409846, valid loss: 0.322107, valid accuracy: 90.76%\n",
      "iteration:  44600, epoch: 115, train loss: 0.394788, valid loss: 0.606165, valid accuracy: 82.93%\n",
      "iteration:  44700, epoch: 115, train loss: 0.427030, valid loss: 0.606165, valid accuracy: 82.93%\n",
      "iteration:  44800, epoch: 115, train loss: 0.358940, valid loss: 0.606165, valid accuracy: 82.93%\n",
      "iteration:  44900, epoch: 115, train loss: 0.388345, valid loss: 0.606165, valid accuracy: 82.93%\n",
      "iteration:  45000, epoch: 116, train loss: 0.356838, valid loss: 0.541448, valid accuracy: 84.64%\n",
      "iteration:  45100, epoch: 116, train loss: 0.394506, valid loss: 0.541448, valid accuracy: 84.64%\n",
      "iteration:  45200, epoch: 116, train loss: 0.404219, valid loss: 0.541448, valid accuracy: 84.64%\n",
      "iteration:  45300, epoch: 116, train loss: 0.356861, valid loss: 0.541448, valid accuracy: 84.64%\n",
      "iteration:  45400, epoch: 117, train loss: 0.422458, valid loss: 0.407300, valid accuracy: 88.53%\n",
      "iteration:  45500, epoch: 117, train loss: 0.451409, valid loss: 0.407300, valid accuracy: 88.53%\n",
      "iteration:  45600, epoch: 117, train loss: 0.437565, valid loss: 0.407300, valid accuracy: 88.53%\n",
      "iteration:  45700, epoch: 117, train loss: 0.331148, valid loss: 0.407300, valid accuracy: 88.53%\n",
      "iteration:  45800, epoch: 118, train loss: 0.390172, valid loss: 0.413366, valid accuracy: 88.06%\n",
      "iteration:  45900, epoch: 118, train loss: 0.387900, valid loss: 0.413366, valid accuracy: 88.06%\n",
      "iteration:  46000, epoch: 118, train loss: 0.339998, valid loss: 0.413366, valid accuracy: 88.06%\n",
      "iteration:  46100, epoch: 118, train loss: 0.409595, valid loss: 0.413366, valid accuracy: 88.06%\n",
      "iteration:  46200, epoch: 119, train loss: 0.360825, valid loss: 0.380798, valid accuracy: 88.97%\n",
      "iteration:  46300, epoch: 119, train loss: 0.358583, valid loss: 0.380798, valid accuracy: 88.97%\n",
      "iteration:  46400, epoch: 119, train loss: 0.450622, valid loss: 0.380798, valid accuracy: 88.97%\n",
      "iteration:  46500, epoch: 119, train loss: 0.348326, valid loss: 0.380798, valid accuracy: 88.97%\n",
      "iteration:  46600, epoch: 120, train loss: 0.355693, valid loss: 0.387282, valid accuracy: 88.57%\n",
      "iteration:  46700, epoch: 120, train loss: 0.389911, valid loss: 0.387282, valid accuracy: 88.57%\n",
      "iteration:  46800, epoch: 120, train loss: 0.396158, valid loss: 0.387282, valid accuracy: 88.57%\n",
      "iteration:  46900, epoch: 120, train loss: 0.397477, valid loss: 0.387282, valid accuracy: 88.57%\n",
      "iteration:  47000, epoch: 121, train loss: 0.345671, valid loss: 0.443533, valid accuracy: 86.83%\n",
      "iteration:  47100, epoch: 121, train loss: 0.341846, valid loss: 0.443533, valid accuracy: 86.83%\n",
      "iteration:  47200, epoch: 121, train loss: 0.399596, valid loss: 0.443533, valid accuracy: 86.83%\n",
      "iteration:  47300, epoch: 121, train loss: 0.374456, valid loss: 0.443533, valid accuracy: 86.83%\n",
      "iteration:  47400, epoch: 122, train loss: 0.459223, valid loss: 0.331409, valid accuracy: 90.33%\n",
      "iteration:  47500, epoch: 122, train loss: 0.381810, valid loss: 0.331409, valid accuracy: 90.33%\n",
      "iteration:  47600, epoch: 122, train loss: 0.412693, valid loss: 0.331409, valid accuracy: 90.33%\n",
      "iteration:  47700, epoch: 122, train loss: 0.420359, valid loss: 0.331409, valid accuracy: 90.33%\n",
      "iteration:  47800, epoch: 123, train loss: 0.379499, valid loss: 0.472507, valid accuracy: 87.08%\n",
      "iteration:  47900, epoch: 123, train loss: 0.350665, valid loss: 0.472507, valid accuracy: 87.08%\n",
      "iteration:  48000, epoch: 123, train loss: 0.395288, valid loss: 0.472507, valid accuracy: 87.08%\n",
      "iteration:  48100, epoch: 124, train loss: 0.451978, valid loss: 0.472173, valid accuracy: 86.81%\n",
      "iteration:  48200, epoch: 124, train loss: 0.419167, valid loss: 0.472173, valid accuracy: 86.81%\n",
      "iteration:  48300, epoch: 124, train loss: 0.336707, valid loss: 0.472173, valid accuracy: 86.81%\n",
      "iteration:  48400, epoch: 124, train loss: 0.435660, valid loss: 0.472173, valid accuracy: 86.81%\n",
      "iteration:  48500, epoch: 125, train loss: 0.392474, valid loss: 0.372747, valid accuracy: 89.96%\n",
      "iteration:  48600, epoch: 125, train loss: 0.397379, valid loss: 0.372747, valid accuracy: 89.96%\n",
      "iteration:  48700, epoch: 125, train loss: 0.361267, valid loss: 0.372747, valid accuracy: 89.96%\n",
      "iteration:  48800, epoch: 125, train loss: 0.342307, valid loss: 0.372747, valid accuracy: 89.96%\n",
      "iteration:  48900, epoch: 126, train loss: 0.381095, valid loss: 0.363968, valid accuracy: 89.56%\n",
      "iteration:  49000, epoch: 126, train loss: 0.372107, valid loss: 0.363968, valid accuracy: 89.56%\n",
      "iteration:  49100, epoch: 126, train loss: 0.356569, valid loss: 0.363968, valid accuracy: 89.56%\n",
      "iteration:  49200, epoch: 126, train loss: 0.362640, valid loss: 0.363968, valid accuracy: 89.56%\n",
      "iteration:  49300, epoch: 127, train loss: 0.348977, valid loss: 0.396733, valid accuracy: 88.52%\n",
      "iteration:  49400, epoch: 127, train loss: 0.347740, valid loss: 0.396733, valid accuracy: 88.52%\n",
      "iteration:  49500, epoch: 127, train loss: 0.428300, valid loss: 0.396733, valid accuracy: 88.52%\n",
      "iteration:  49600, epoch: 127, train loss: 0.342076, valid loss: 0.396733, valid accuracy: 88.52%\n",
      "iteration:  49700, epoch: 128, train loss: 0.428868, valid loss: 0.452879, valid accuracy: 87.35%\n",
      "iteration:  49800, epoch: 128, train loss: 0.336911, valid loss: 0.452879, valid accuracy: 87.35%\n",
      "iteration:  49900, epoch: 128, train loss: 0.364173, valid loss: 0.452879, valid accuracy: 87.35%\n",
      "iteration:  50000, epoch: 128, train loss: 0.373977, valid loss: 0.452879, valid accuracy: 87.35%\n",
      "iteration:  50100, epoch: 129, train loss: 0.347044, valid loss: 0.459075, valid accuracy: 87.39%\n",
      "iteration:  50200, epoch: 129, train loss: 0.437771, valid loss: 0.459075, valid accuracy: 87.39%\n",
      "iteration:  50300, epoch: 129, train loss: 0.379327, valid loss: 0.459075, valid accuracy: 87.39%\n",
      "iteration:  50400, epoch: 129, train loss: 0.387045, valid loss: 0.459075, valid accuracy: 87.39%\n",
      "iteration:  50500, epoch: 130, train loss: 0.466804, valid loss: 0.403165, valid accuracy: 88.75%\n",
      "iteration:  50600, epoch: 130, train loss: 0.353779, valid loss: 0.403165, valid accuracy: 88.75%\n",
      "iteration:  50700, epoch: 130, train loss: 0.400877, valid loss: 0.403165, valid accuracy: 88.75%\n",
      "iteration:  50800, epoch: 130, train loss: 0.334878, valid loss: 0.403165, valid accuracy: 88.75%\n",
      "iteration:  50900, epoch: 131, train loss: 0.393906, valid loss: 0.394189, valid accuracy: 88.75%\n",
      "iteration:  51000, epoch: 131, train loss: 0.387189, valid loss: 0.394189, valid accuracy: 88.75%\n",
      "iteration:  51100, epoch: 131, train loss: 0.411789, valid loss: 0.394189, valid accuracy: 88.75%\n",
      "iteration:  51200, epoch: 131, train loss: 0.358439, valid loss: 0.394189, valid accuracy: 88.75%\n",
      "iteration:  51300, epoch: 132, train loss: 0.318048, valid loss: 0.488817, valid accuracy: 86.16%\n",
      "iteration:  51400, epoch: 132, train loss: 0.375893, valid loss: 0.488817, valid accuracy: 86.16%\n",
      "iteration:  51500, epoch: 132, train loss: 0.375358, valid loss: 0.488817, valid accuracy: 86.16%\n",
      "iteration:  51600, epoch: 132, train loss: 0.352224, valid loss: 0.488817, valid accuracy: 86.16%\n",
      "iteration:  51700, epoch: 133, train loss: 0.351732, valid loss: 0.415240, valid accuracy: 88.20%\n",
      "iteration:  51800, epoch: 133, train loss: 0.428486, valid loss: 0.415240, valid accuracy: 88.20%\n",
      "iteration:  51900, epoch: 133, train loss: 0.336538, valid loss: 0.415240, valid accuracy: 88.20%\n",
      "iteration:  52000, epoch: 133, train loss: 0.363646, valid loss: 0.415240, valid accuracy: 88.20%\n",
      "iteration:  52100, epoch: 134, train loss: 0.386683, valid loss: 0.397437, valid accuracy: 88.84%\n",
      "iteration:  52200, epoch: 134, train loss: 0.361064, valid loss: 0.397437, valid accuracy: 88.84%\n",
      "iteration:  52300, epoch: 134, train loss: 0.377223, valid loss: 0.397437, valid accuracy: 88.84%\n",
      "iteration:  52400, epoch: 135, train loss: 0.336265, valid loss: 0.379005, valid accuracy: 89.25%\n",
      "iteration:  52500, epoch: 135, train loss: 0.372951, valid loss: 0.379005, valid accuracy: 89.25%\n",
      "iteration:  52600, epoch: 135, train loss: 0.354373, valid loss: 0.379005, valid accuracy: 89.25%\n",
      "iteration:  52700, epoch: 135, train loss: 0.350681, valid loss: 0.379005, valid accuracy: 89.25%\n",
      "iteration:  52800, epoch: 136, train loss: 0.336611, valid loss: 0.358971, valid accuracy: 89.45%\n",
      "iteration:  52900, epoch: 136, train loss: 0.375728, valid loss: 0.358971, valid accuracy: 89.45%\n",
      "iteration:  53000, epoch: 136, train loss: 0.377408, valid loss: 0.358971, valid accuracy: 89.45%\n",
      "iteration:  53100, epoch: 136, train loss: 0.414387, valid loss: 0.358971, valid accuracy: 89.45%\n",
      "iteration:  53200, epoch: 137, train loss: 0.364385, valid loss: 0.333268, valid accuracy: 90.50%\n",
      "iteration:  53300, epoch: 137, train loss: 0.360787, valid loss: 0.333268, valid accuracy: 90.50%\n",
      "iteration:  53400, epoch: 137, train loss: 0.352506, valid loss: 0.333268, valid accuracy: 90.50%\n",
      "iteration:  53500, epoch: 137, train loss: 0.386480, valid loss: 0.333268, valid accuracy: 90.50%\n",
      "iteration:  53600, epoch: 138, train loss: 0.349982, valid loss: 0.501525, valid accuracy: 86.53%\n",
      "iteration:  53700, epoch: 138, train loss: 0.352785, valid loss: 0.501525, valid accuracy: 86.53%\n",
      "iteration:  53800, epoch: 138, train loss: 0.403443, valid loss: 0.501525, valid accuracy: 86.53%\n",
      "iteration:  53900, epoch: 138, train loss: 0.400661, valid loss: 0.501525, valid accuracy: 86.53%\n",
      "iteration:  54000, epoch: 139, train loss: 0.312174, valid loss: 0.341013, valid accuracy: 90.21%\n",
      "iteration:  54100, epoch: 139, train loss: 0.342310, valid loss: 0.341013, valid accuracy: 90.21%\n",
      "iteration:  54200, epoch: 139, train loss: 0.349089, valid loss: 0.341013, valid accuracy: 90.21%\n",
      "iteration:  54300, epoch: 139, train loss: 0.324075, valid loss: 0.341013, valid accuracy: 90.21%\n",
      "iteration:  54400, epoch: 140, train loss: 0.322536, valid loss: 0.329843, valid accuracy: 90.90%\n",
      "iteration:  54500, epoch: 140, train loss: 0.336633, valid loss: 0.329843, valid accuracy: 90.90%\n",
      "iteration:  54600, epoch: 140, train loss: 0.347866, valid loss: 0.329843, valid accuracy: 90.90%\n",
      "iteration:  54700, epoch: 140, train loss: 0.425654, valid loss: 0.329843, valid accuracy: 90.90%\n",
      "iteration:  54800, epoch: 141, train loss: 0.338546, valid loss: 0.384299, valid accuracy: 89.32%\n",
      "iteration:  54900, epoch: 141, train loss: 0.357287, valid loss: 0.384299, valid accuracy: 89.32%\n",
      "iteration:  55000, epoch: 141, train loss: 0.405424, valid loss: 0.384299, valid accuracy: 89.32%\n",
      "iteration:  55100, epoch: 141, train loss: 0.416321, valid loss: 0.384299, valid accuracy: 89.32%\n",
      "iteration:  55200, epoch: 142, train loss: 0.330157, valid loss: 0.433802, valid accuracy: 87.25%\n",
      "iteration:  55300, epoch: 142, train loss: 0.325755, valid loss: 0.433802, valid accuracy: 87.25%\n",
      "iteration:  55400, epoch: 142, train loss: 0.358509, valid loss: 0.433802, valid accuracy: 87.25%\n",
      "iteration:  55500, epoch: 142, train loss: 0.465577, valid loss: 0.433802, valid accuracy: 87.25%\n",
      "iteration:  55600, epoch: 143, train loss: 0.360103, valid loss: 0.392594, valid accuracy: 88.53%\n",
      "iteration:  55700, epoch: 143, train loss: 0.396802, valid loss: 0.392594, valid accuracy: 88.53%\n",
      "iteration:  55800, epoch: 143, train loss: 0.373855, valid loss: 0.392594, valid accuracy: 88.53%\n",
      "iteration:  55900, epoch: 143, train loss: 0.343829, valid loss: 0.392594, valid accuracy: 88.53%\n",
      "iteration:  56000, epoch: 144, train loss: 0.388937, valid loss: 0.377783, valid accuracy: 89.76%\n",
      "iteration:  56100, epoch: 144, train loss: 0.358215, valid loss: 0.377783, valid accuracy: 89.76%\n",
      "iteration:  56200, epoch: 144, train loss: 0.427918, valid loss: 0.377783, valid accuracy: 89.76%\n",
      "iteration:  56300, epoch: 144, train loss: 0.376168, valid loss: 0.377783, valid accuracy: 89.76%\n",
      "iteration:  56400, epoch: 145, train loss: 0.354451, valid loss: 0.368068, valid accuracy: 90.07%\n",
      "iteration:  56500, epoch: 145, train loss: 0.413986, valid loss: 0.368068, valid accuracy: 90.07%\n",
      "iteration:  56600, epoch: 145, train loss: 0.362658, valid loss: 0.368068, valid accuracy: 90.07%\n",
      "iteration:  56700, epoch: 146, train loss: 0.325098, valid loss: 0.373398, valid accuracy: 89.68%\n",
      "iteration:  56800, epoch: 146, train loss: 0.336591, valid loss: 0.373398, valid accuracy: 89.68%\n",
      "iteration:  56900, epoch: 146, train loss: 0.366904, valid loss: 0.373398, valid accuracy: 89.68%\n",
      "iteration:  57000, epoch: 146, train loss: 0.349079, valid loss: 0.373398, valid accuracy: 89.68%\n",
      "iteration:  57100, epoch: 147, train loss: 0.396777, valid loss: 0.396612, valid accuracy: 88.79%\n",
      "iteration:  57200, epoch: 147, train loss: 0.355859, valid loss: 0.396612, valid accuracy: 88.79%\n",
      "iteration:  57300, epoch: 147, train loss: 0.409511, valid loss: 0.396612, valid accuracy: 88.79%\n",
      "iteration:  57400, epoch: 147, train loss: 0.344882, valid loss: 0.396612, valid accuracy: 88.79%\n",
      "iteration:  57500, epoch: 148, train loss: 0.360617, valid loss: 0.347505, valid accuracy: 90.60%\n",
      "iteration:  57600, epoch: 148, train loss: 0.362185, valid loss: 0.347505, valid accuracy: 90.60%\n",
      "iteration:  57700, epoch: 148, train loss: 0.346259, valid loss: 0.347505, valid accuracy: 90.60%\n",
      "iteration:  57800, epoch: 148, train loss: 0.329429, valid loss: 0.347505, valid accuracy: 90.60%\n",
      "iteration:  57900, epoch: 149, train loss: 0.377683, valid loss: 0.405843, valid accuracy: 88.76%\n",
      "iteration:  58000, epoch: 149, train loss: 0.392743, valid loss: 0.405843, valid accuracy: 88.76%\n",
      "iteration:  58100, epoch: 149, train loss: 0.371780, valid loss: 0.405843, valid accuracy: 88.76%\n",
      "iteration:  58200, epoch: 149, train loss: 0.366609, valid loss: 0.405843, valid accuracy: 88.76%\n",
      "iteration:  58300, epoch: 150, train loss: 0.358952, valid loss: 0.344374, valid accuracy: 90.26%\n",
      "iteration:  58400, epoch: 150, train loss: 0.361018, valid loss: 0.344374, valid accuracy: 90.26%\n",
      "iteration:  58500, epoch: 150, train loss: 0.338239, valid loss: 0.344374, valid accuracy: 90.26%\n",
      "iteration:  58600, epoch: 150, train loss: 0.322877, valid loss: 0.344374, valid accuracy: 90.26%\n",
      "iteration:  58700, epoch: 151, train loss: 0.370289, valid loss: 0.338931, valid accuracy: 90.59%\n",
      "iteration:  58800, epoch: 151, train loss: 0.311007, valid loss: 0.338931, valid accuracy: 90.59%\n",
      "iteration:  58900, epoch: 151, train loss: 0.315902, valid loss: 0.338931, valid accuracy: 90.59%\n",
      "iteration:  59000, epoch: 151, train loss: 0.348488, valid loss: 0.338931, valid accuracy: 90.59%\n",
      "iteration:  59100, epoch: 152, train loss: 0.382179, valid loss: 0.322064, valid accuracy: 91.13%\n",
      "iteration:  59200, epoch: 152, train loss: 0.313419, valid loss: 0.322064, valid accuracy: 91.13%\n",
      "iteration:  59300, epoch: 152, train loss: 0.394046, valid loss: 0.322064, valid accuracy: 91.13%\n",
      "iteration:  59400, epoch: 152, train loss: 0.363184, valid loss: 0.322064, valid accuracy: 91.13%\n",
      "iteration:  59500, epoch: 153, train loss: 0.401328, valid loss: 0.399252, valid accuracy: 88.97%\n",
      "iteration:  59600, epoch: 153, train loss: 0.360326, valid loss: 0.399252, valid accuracy: 88.97%\n",
      "iteration:  59700, epoch: 153, train loss: 0.330645, valid loss: 0.399252, valid accuracy: 88.97%\n",
      "iteration:  59800, epoch: 153, train loss: 0.394650, valid loss: 0.399252, valid accuracy: 88.97%\n",
      "iteration:  59900, epoch: 154, train loss: 0.314379, valid loss: 0.451218, valid accuracy: 88.17%\n",
      "iteration:  60000, epoch: 154, train loss: 0.368813, valid loss: 0.451218, valid accuracy: 88.17%\n",
      "iteration:  60100, epoch: 154, train loss: 0.327599, valid loss: 0.451218, valid accuracy: 88.17%\n",
      "iteration:  60200, epoch: 154, train loss: 0.346835, valid loss: 0.451218, valid accuracy: 88.17%\n",
      "iteration:  60300, epoch: 155, train loss: 0.343484, valid loss: 0.437552, valid accuracy: 88.44%\n",
      "iteration:  60400, epoch: 155, train loss: 0.367898, valid loss: 0.437552, valid accuracy: 88.44%\n",
      "iteration:  60500, epoch: 155, train loss: 0.339389, valid loss: 0.437552, valid accuracy: 88.44%\n",
      "iteration:  60600, epoch: 155, train loss: 0.349177, valid loss: 0.437552, valid accuracy: 88.44%\n",
      "iteration:  60700, epoch: 156, train loss: 0.339286, valid loss: 0.416397, valid accuracy: 88.89%\n",
      "iteration:  60800, epoch: 156, train loss: 0.345504, valid loss: 0.416397, valid accuracy: 88.89%\n",
      "iteration:  60900, epoch: 156, train loss: 0.397446, valid loss: 0.416397, valid accuracy: 88.89%\n",
      "iteration:  61000, epoch: 157, train loss: 0.363958, valid loss: 0.395258, valid accuracy: 89.13%\n",
      "iteration:  61100, epoch: 157, train loss: 0.336209, valid loss: 0.395258, valid accuracy: 89.13%\n",
      "iteration:  61200, epoch: 157, train loss: 0.394976, valid loss: 0.395258, valid accuracy: 89.13%\n",
      "iteration:  61300, epoch: 157, train loss: 0.347263, valid loss: 0.395258, valid accuracy: 89.13%\n",
      "iteration:  61400, epoch: 158, train loss: 0.351898, valid loss: 0.377103, valid accuracy: 89.52%\n",
      "iteration:  61500, epoch: 158, train loss: 0.400920, valid loss: 0.377103, valid accuracy: 89.52%\n",
      "iteration:  61600, epoch: 158, train loss: 0.326360, valid loss: 0.377103, valid accuracy: 89.52%\n",
      "iteration:  61700, epoch: 158, train loss: 0.346145, valid loss: 0.377103, valid accuracy: 89.52%\n",
      "iteration:  61800, epoch: 159, train loss: 0.353830, valid loss: 0.333655, valid accuracy: 90.84%\n",
      "iteration:  61900, epoch: 159, train loss: 0.372639, valid loss: 0.333655, valid accuracy: 90.84%\n",
      "iteration:  62000, epoch: 159, train loss: 0.398479, valid loss: 0.333655, valid accuracy: 90.84%\n",
      "iteration:  62100, epoch: 159, train loss: 0.394103, valid loss: 0.333655, valid accuracy: 90.84%\n",
      "iteration:  62200, epoch: 160, train loss: 0.393856, valid loss: 0.362811, valid accuracy: 89.95%\n",
      "iteration:  62300, epoch: 160, train loss: 0.339041, valid loss: 0.362811, valid accuracy: 89.95%\n",
      "iteration:  62400, epoch: 160, train loss: 0.323388, valid loss: 0.362811, valid accuracy: 89.95%\n",
      "iteration:  62500, epoch: 160, train loss: 0.344843, valid loss: 0.362811, valid accuracy: 89.95%\n",
      "iteration:  62600, epoch: 161, train loss: 0.321734, valid loss: 0.367814, valid accuracy: 90.12%\n",
      "iteration:  62700, epoch: 161, train loss: 0.378015, valid loss: 0.367814, valid accuracy: 90.12%\n",
      "iteration:  62800, epoch: 161, train loss: 0.327383, valid loss: 0.367814, valid accuracy: 90.12%\n",
      "iteration:  62900, epoch: 161, train loss: 0.331697, valid loss: 0.367814, valid accuracy: 90.12%\n",
      "iteration:  63000, epoch: 162, train loss: 0.322237, valid loss: 0.351441, valid accuracy: 90.69%\n",
      "iteration:  63100, epoch: 162, train loss: 0.343623, valid loss: 0.351441, valid accuracy: 90.69%\n",
      "iteration:  63200, epoch: 162, train loss: 0.400539, valid loss: 0.351441, valid accuracy: 90.69%\n",
      "iteration:  63300, epoch: 162, train loss: 0.394307, valid loss: 0.351441, valid accuracy: 90.69%\n",
      "iteration:  63400, epoch: 163, train loss: 0.368269, valid loss: 0.368792, valid accuracy: 90.00%\n",
      "iteration:  63500, epoch: 163, train loss: 0.333399, valid loss: 0.368792, valid accuracy: 90.00%\n",
      "iteration:  63600, epoch: 163, train loss: 0.317392, valid loss: 0.368792, valid accuracy: 90.00%\n",
      "iteration:  63700, epoch: 163, train loss: 0.343121, valid loss: 0.368792, valid accuracy: 90.00%\n",
      "iteration:  63800, epoch: 164, train loss: 0.375571, valid loss: 0.325974, valid accuracy: 91.10%\n",
      "iteration:  63900, epoch: 164, train loss: 0.307537, valid loss: 0.325974, valid accuracy: 91.10%\n",
      "iteration:  64000, epoch: 164, train loss: 0.335964, valid loss: 0.325974, valid accuracy: 91.10%\n",
      "iteration:  64100, epoch: 164, train loss: 0.349440, valid loss: 0.325974, valid accuracy: 91.10%\n",
      "iteration:  64200, epoch: 165, train loss: 0.342448, valid loss: 0.432593, valid accuracy: 88.68%\n",
      "iteration:  64300, epoch: 165, train loss: 0.370487, valid loss: 0.432593, valid accuracy: 88.68%\n",
      "iteration:  64400, epoch: 165, train loss: 0.303162, valid loss: 0.432593, valid accuracy: 88.68%\n",
      "iteration:  64500, epoch: 165, train loss: 0.344400, valid loss: 0.432593, valid accuracy: 88.68%\n",
      "iteration:  64600, epoch: 166, train loss: 0.334262, valid loss: 0.315639, valid accuracy: 91.36%\n",
      "iteration:  64700, epoch: 166, train loss: 0.351743, valid loss: 0.315639, valid accuracy: 91.36%\n",
      "iteration:  64800, epoch: 166, train loss: 0.301613, valid loss: 0.315639, valid accuracy: 91.36%\n",
      "iteration:  64900, epoch: 166, train loss: 0.321286, valid loss: 0.315639, valid accuracy: 91.36%\n",
      "iteration:  65000, epoch: 167, train loss: 0.339677, valid loss: 0.355136, valid accuracy: 90.28%\n",
      "iteration:  65100, epoch: 167, train loss: 0.383692, valid loss: 0.355136, valid accuracy: 90.28%\n",
      "iteration:  65200, epoch: 167, train loss: 0.422829, valid loss: 0.355136, valid accuracy: 90.28%\n",
      "iteration:  65300, epoch: 168, train loss: 0.314020, valid loss: 0.327606, valid accuracy: 91.18%\n",
      "iteration:  65400, epoch: 168, train loss: 0.324825, valid loss: 0.327606, valid accuracy: 91.18%\n",
      "iteration:  65500, epoch: 168, train loss: 0.399413, valid loss: 0.327606, valid accuracy: 91.18%\n",
      "iteration:  65600, epoch: 168, train loss: 0.339263, valid loss: 0.327606, valid accuracy: 91.18%\n",
      "iteration:  65700, epoch: 169, train loss: 0.364187, valid loss: 0.439377, valid accuracy: 88.67%\n",
      "iteration:  65800, epoch: 169, train loss: 0.319705, valid loss: 0.439377, valid accuracy: 88.67%\n",
      "iteration:  65900, epoch: 169, train loss: 0.346168, valid loss: 0.439377, valid accuracy: 88.67%\n",
      "iteration:  66000, epoch: 169, train loss: 0.351685, valid loss: 0.439377, valid accuracy: 88.67%\n",
      "iteration:  66100, epoch: 170, train loss: 0.357749, valid loss: 0.361858, valid accuracy: 89.92%\n",
      "iteration:  66200, epoch: 170, train loss: 0.328496, valid loss: 0.361858, valid accuracy: 89.92%\n",
      "iteration:  66300, epoch: 170, train loss: 0.426795, valid loss: 0.361858, valid accuracy: 89.92%\n",
      "iteration:  66400, epoch: 170, train loss: 0.376104, valid loss: 0.361858, valid accuracy: 89.92%\n",
      "iteration:  66500, epoch: 171, train loss: 0.302938, valid loss: 0.339974, valid accuracy: 90.77%\n",
      "iteration:  66600, epoch: 171, train loss: 0.360826, valid loss: 0.339974, valid accuracy: 90.77%\n",
      "iteration:  66700, epoch: 171, train loss: 0.330345, valid loss: 0.339974, valid accuracy: 90.77%\n",
      "iteration:  66800, epoch: 171, train loss: 0.343510, valid loss: 0.339974, valid accuracy: 90.77%\n",
      "iteration:  66900, epoch: 172, train loss: 0.372870, valid loss: 0.342704, valid accuracy: 90.77%\n",
      "iteration:  67000, epoch: 172, train loss: 0.333462, valid loss: 0.342704, valid accuracy: 90.77%\n",
      "iteration:  67100, epoch: 172, train loss: 0.320575, valid loss: 0.342704, valid accuracy: 90.77%\n",
      "iteration:  67200, epoch: 172, train loss: 0.353693, valid loss: 0.342704, valid accuracy: 90.77%\n",
      "iteration:  67300, epoch: 173, train loss: 0.359326, valid loss: 0.306935, valid accuracy: 91.49%\n",
      "iteration:  67400, epoch: 173, train loss: 0.347787, valid loss: 0.306935, valid accuracy: 91.49%\n",
      "iteration:  67500, epoch: 173, train loss: 0.307694, valid loss: 0.306935, valid accuracy: 91.49%\n",
      "iteration:  67600, epoch: 173, train loss: 0.381711, valid loss: 0.306935, valid accuracy: 91.49%\n",
      "iteration:  67700, epoch: 174, train loss: 0.340114, valid loss: 0.333772, valid accuracy: 91.01%\n",
      "iteration:  67800, epoch: 174, train loss: 0.317624, valid loss: 0.333772, valid accuracy: 91.01%\n",
      "iteration:  67900, epoch: 174, train loss: 0.325371, valid loss: 0.333772, valid accuracy: 91.01%\n",
      "iteration:  68000, epoch: 174, train loss: 0.363291, valid loss: 0.333772, valid accuracy: 91.01%\n",
      "iteration:  68100, epoch: 175, train loss: 0.415540, valid loss: 0.316651, valid accuracy: 91.53%\n",
      "iteration:  68200, epoch: 175, train loss: 0.321427, valid loss: 0.316651, valid accuracy: 91.53%\n",
      "iteration:  68300, epoch: 175, train loss: 0.303415, valid loss: 0.316651, valid accuracy: 91.53%\n",
      "iteration:  68400, epoch: 175, train loss: 0.318532, valid loss: 0.316651, valid accuracy: 91.53%\n",
      "iteration:  68500, epoch: 176, train loss: 0.313722, valid loss: 0.320682, valid accuracy: 91.25%\n",
      "iteration:  68600, epoch: 176, train loss: 0.306691, valid loss: 0.320682, valid accuracy: 91.25%\n",
      "iteration:  68700, epoch: 176, train loss: 0.309475, valid loss: 0.320682, valid accuracy: 91.25%\n",
      "iteration:  68800, epoch: 176, train loss: 0.341293, valid loss: 0.320682, valid accuracy: 91.25%\n",
      "iteration:  68900, epoch: 177, train loss: 0.323561, valid loss: 0.347730, valid accuracy: 90.64%\n",
      "iteration:  69000, epoch: 177, train loss: 0.352974, valid loss: 0.347730, valid accuracy: 90.64%\n",
      "iteration:  69100, epoch: 177, train loss: 0.313154, valid loss: 0.347730, valid accuracy: 90.64%\n",
      "iteration:  69200, epoch: 177, train loss: 0.302298, valid loss: 0.347730, valid accuracy: 90.64%\n",
      "iteration:  69300, epoch: 178, train loss: 0.324699, valid loss: 0.327201, valid accuracy: 91.09%\n",
      "iteration:  69400, epoch: 178, train loss: 0.394229, valid loss: 0.327201, valid accuracy: 91.09%\n",
      "iteration:  69500, epoch: 178, train loss: 0.330763, valid loss: 0.327201, valid accuracy: 91.09%\n",
      "iteration:  69600, epoch: 179, train loss: 0.316565, valid loss: 0.337885, valid accuracy: 90.81%\n",
      "iteration:  69700, epoch: 179, train loss: 0.337574, valid loss: 0.337885, valid accuracy: 90.81%\n",
      "iteration:  69800, epoch: 179, train loss: 0.345545, valid loss: 0.337885, valid accuracy: 90.81%\n",
      "iteration:  69900, epoch: 179, train loss: 0.317670, valid loss: 0.337885, valid accuracy: 90.81%\n",
      "iteration:  70000, epoch: 180, train loss: 0.347091, valid loss: 0.373237, valid accuracy: 90.15%\n",
      "iteration:  70100, epoch: 180, train loss: 0.362216, valid loss: 0.373237, valid accuracy: 90.15%\n",
      "iteration:  70200, epoch: 180, train loss: 0.387324, valid loss: 0.373237, valid accuracy: 90.15%\n",
      "iteration:  70300, epoch: 180, train loss: 0.321974, valid loss: 0.373237, valid accuracy: 90.15%\n",
      "iteration:  70400, epoch: 181, train loss: 0.320517, valid loss: 0.313321, valid accuracy: 91.80%\n",
      "iteration:  70500, epoch: 181, train loss: 0.346019, valid loss: 0.313321, valid accuracy: 91.80%\n",
      "iteration:  70600, epoch: 181, train loss: 0.391213, valid loss: 0.313321, valid accuracy: 91.80%\n",
      "iteration:  70700, epoch: 181, train loss: 0.332662, valid loss: 0.313321, valid accuracy: 91.80%\n",
      "iteration:  70800, epoch: 182, train loss: 0.329252, valid loss: 0.337222, valid accuracy: 90.90%\n",
      "iteration:  70900, epoch: 182, train loss: 0.302393, valid loss: 0.337222, valid accuracy: 90.90%\n",
      "iteration:  71000, epoch: 182, train loss: 0.358798, valid loss: 0.337222, valid accuracy: 90.90%\n",
      "iteration:  71100, epoch: 182, train loss: 0.328640, valid loss: 0.337222, valid accuracy: 90.90%\n",
      "iteration:  71200, epoch: 183, train loss: 0.307278, valid loss: 0.352258, valid accuracy: 90.90%\n",
      "iteration:  71300, epoch: 183, train loss: 0.357272, valid loss: 0.352258, valid accuracy: 90.90%\n",
      "iteration:  71400, epoch: 183, train loss: 0.337602, valid loss: 0.352258, valid accuracy: 90.90%\n",
      "iteration:  71500, epoch: 183, train loss: 0.309360, valid loss: 0.352258, valid accuracy: 90.90%\n",
      "iteration:  71600, epoch: 184, train loss: 0.301750, valid loss: 0.318621, valid accuracy: 91.78%\n",
      "iteration:  71700, epoch: 184, train loss: 0.321547, valid loss: 0.318621, valid accuracy: 91.78%\n",
      "iteration:  71800, epoch: 184, train loss: 0.326258, valid loss: 0.318621, valid accuracy: 91.78%\n",
      "iteration:  71900, epoch: 184, train loss: 0.298344, valid loss: 0.318621, valid accuracy: 91.78%\n",
      "iteration:  72000, epoch: 185, train loss: 0.363285, valid loss: 0.309724, valid accuracy: 91.94%\n",
      "iteration:  72100, epoch: 185, train loss: 0.337302, valid loss: 0.309724, valid accuracy: 91.94%\n",
      "iteration:  72200, epoch: 185, train loss: 0.303643, valid loss: 0.309724, valid accuracy: 91.94%\n",
      "iteration:  72300, epoch: 185, train loss: 0.353705, valid loss: 0.309724, valid accuracy: 91.94%\n",
      "iteration:  72400, epoch: 186, train loss: 0.313312, valid loss: 0.314611, valid accuracy: 91.63%\n",
      "iteration:  72500, epoch: 186, train loss: 0.311075, valid loss: 0.314611, valid accuracy: 91.63%\n",
      "iteration:  72600, epoch: 186, train loss: 0.344352, valid loss: 0.314611, valid accuracy: 91.63%\n",
      "iteration:  72700, epoch: 186, train loss: 0.350495, valid loss: 0.314611, valid accuracy: 91.63%\n",
      "iteration:  72800, epoch: 187, train loss: 0.371810, valid loss: 0.330596, valid accuracy: 91.06%\n",
      "iteration:  72900, epoch: 187, train loss: 0.349646, valid loss: 0.330596, valid accuracy: 91.06%\n",
      "iteration:  73000, epoch: 187, train loss: 0.355042, valid loss: 0.330596, valid accuracy: 91.06%\n",
      "iteration:  73100, epoch: 187, train loss: 0.332158, valid loss: 0.330596, valid accuracy: 91.06%\n",
      "iteration:  73200, epoch: 188, train loss: 0.351448, valid loss: 0.345838, valid accuracy: 90.52%\n",
      "iteration:  73300, epoch: 188, train loss: 0.308854, valid loss: 0.345838, valid accuracy: 90.52%\n",
      "iteration:  73400, epoch: 188, train loss: 0.301473, valid loss: 0.345838, valid accuracy: 90.52%\n",
      "iteration:  73500, epoch: 188, train loss: 0.376408, valid loss: 0.345838, valid accuracy: 90.52%\n",
      "iteration:  73600, epoch: 189, train loss: 0.334968, valid loss: 0.324237, valid accuracy: 91.52%\n",
      "iteration:  73700, epoch: 189, train loss: 0.326905, valid loss: 0.324237, valid accuracy: 91.52%\n",
      "iteration:  73800, epoch: 189, train loss: 0.306084, valid loss: 0.324237, valid accuracy: 91.52%\n",
      "iteration:  73900, epoch: 190, train loss: 0.325934, valid loss: 0.330940, valid accuracy: 91.15%\n",
      "iteration:  74000, epoch: 190, train loss: 0.353889, valid loss: 0.330940, valid accuracy: 91.15%\n",
      "iteration:  74100, epoch: 190, train loss: 0.365084, valid loss: 0.330940, valid accuracy: 91.15%\n",
      "iteration:  74200, epoch: 190, train loss: 0.299120, valid loss: 0.330940, valid accuracy: 91.15%\n",
      "iteration:  74300, epoch: 191, train loss: 0.303668, valid loss: 0.319894, valid accuracy: 91.35%\n",
      "iteration:  74400, epoch: 191, train loss: 0.335572, valid loss: 0.319894, valid accuracy: 91.35%\n",
      "iteration:  74500, epoch: 191, train loss: 0.330704, valid loss: 0.319894, valid accuracy: 91.35%\n",
      "iteration:  74600, epoch: 191, train loss: 0.325758, valid loss: 0.319894, valid accuracy: 91.35%\n",
      "iteration:  74700, epoch: 192, train loss: 0.368263, valid loss: 0.329035, valid accuracy: 91.50%\n",
      "iteration:  74800, epoch: 192, train loss: 0.314787, valid loss: 0.329035, valid accuracy: 91.50%\n",
      "iteration:  74900, epoch: 192, train loss: 0.304439, valid loss: 0.329035, valid accuracy: 91.50%\n",
      "iteration:  75000, epoch: 192, train loss: 0.333975, valid loss: 0.329035, valid accuracy: 91.50%\n",
      "iteration:  75100, epoch: 193, train loss: 0.316839, valid loss: 0.346109, valid accuracy: 90.85%\n",
      "iteration:  75200, epoch: 193, train loss: 0.324778, valid loss: 0.346109, valid accuracy: 90.85%\n",
      "iteration:  75300, epoch: 193, train loss: 0.308722, valid loss: 0.346109, valid accuracy: 90.85%\n",
      "iteration:  75400, epoch: 193, train loss: 0.304379, valid loss: 0.346109, valid accuracy: 90.85%\n",
      "iteration:  75500, epoch: 194, train loss: 0.294361, valid loss: 0.312069, valid accuracy: 91.52%\n",
      "iteration:  75600, epoch: 194, train loss: 0.289908, valid loss: 0.312069, valid accuracy: 91.52%\n",
      "iteration:  75700, epoch: 194, train loss: 0.364909, valid loss: 0.312069, valid accuracy: 91.52%\n",
      "iteration:  75800, epoch: 194, train loss: 0.344929, valid loss: 0.312069, valid accuracy: 91.52%\n",
      "iteration:  75900, epoch: 195, train loss: 0.342237, valid loss: 0.304342, valid accuracy: 92.04%\n",
      "iteration:  76000, epoch: 195, train loss: 0.323917, valid loss: 0.304342, valid accuracy: 92.04%\n",
      "iteration:  76100, epoch: 195, train loss: 0.294116, valid loss: 0.304342, valid accuracy: 92.04%\n",
      "iteration:  76200, epoch: 195, train loss: 0.332618, valid loss: 0.304342, valid accuracy: 92.04%\n",
      "iteration:  76300, epoch: 196, train loss: 0.336727, valid loss: 0.366900, valid accuracy: 90.68%\n",
      "iteration:  76400, epoch: 196, train loss: 0.370014, valid loss: 0.366900, valid accuracy: 90.68%\n",
      "iteration:  76500, epoch: 196, train loss: 0.399959, valid loss: 0.366900, valid accuracy: 90.68%\n",
      "iteration:  76600, epoch: 196, train loss: 0.324195, valid loss: 0.366900, valid accuracy: 90.68%\n",
      "iteration:  76700, epoch: 197, train loss: 0.342703, valid loss: 0.322884, valid accuracy: 91.35%\n",
      "iteration:  76800, epoch: 197, train loss: 0.336899, valid loss: 0.322884, valid accuracy: 91.35%\n",
      "iteration:  76900, epoch: 197, train loss: 0.293751, valid loss: 0.322884, valid accuracy: 91.35%\n",
      "iteration:  77000, epoch: 197, train loss: 0.308788, valid loss: 0.322884, valid accuracy: 91.35%\n",
      "iteration:  77100, epoch: 198, train loss: 0.321804, valid loss: 0.321354, valid accuracy: 91.46%\n",
      "iteration:  77200, epoch: 198, train loss: 0.299458, valid loss: 0.321354, valid accuracy: 91.46%\n",
      "iteration:  77300, epoch: 198, train loss: 0.316911, valid loss: 0.321354, valid accuracy: 91.46%\n",
      "iteration:  77400, epoch: 198, train loss: 0.317720, valid loss: 0.321354, valid accuracy: 91.46%\n",
      "iteration:  77500, epoch: 199, train loss: 0.329119, valid loss: 0.333219, valid accuracy: 91.43%\n",
      "iteration:  77600, epoch: 199, train loss: 0.295568, valid loss: 0.333219, valid accuracy: 91.43%\n",
      "iteration:  77700, epoch: 199, train loss: 0.342738, valid loss: 0.333219, valid accuracy: 91.43%\n",
      "iteration:  77800, epoch: 199, train loss: 0.343291, valid loss: 0.333219, valid accuracy: 91.43%\n",
      "iteration:  77900, epoch: 200, train loss: 0.309623, valid loss: 0.328289, valid accuracy: 91.35%\n",
      "iteration:  78000, epoch: 200, train loss: 0.308639, valid loss: 0.328289, valid accuracy: 91.35%\n",
      "iteration:  78100, epoch: 200, train loss: 0.364841, valid loss: 0.328289, valid accuracy: 91.35%\n",
      "iteration:  78200, epoch: 200, train loss: 0.301835, valid loss: 0.328289, valid accuracy: 91.35%\n",
      "iteration:  78300, epoch: 201, train loss: 0.317956, valid loss: 0.328091, valid accuracy: 91.45%\n",
      "iteration:  78400, epoch: 201, train loss: 0.302385, valid loss: 0.328091, valid accuracy: 91.45%\n",
      "iteration:  78500, epoch: 201, train loss: 0.329376, valid loss: 0.328091, valid accuracy: 91.45%\n",
      "iteration:  78600, epoch: 202, train loss: 0.291391, valid loss: 0.313737, valid accuracy: 91.65%\n",
      "iteration:  78700, epoch: 202, train loss: 0.317319, valid loss: 0.313737, valid accuracy: 91.65%\n",
      "iteration:  78800, epoch: 202, train loss: 0.337046, valid loss: 0.313737, valid accuracy: 91.65%\n",
      "iteration:  78900, epoch: 202, train loss: 0.334258, valid loss: 0.313737, valid accuracy: 91.65%\n",
      "iteration:  79000, epoch: 203, train loss: 0.320875, valid loss: 0.297132, valid accuracy: 92.25%\n",
      "iteration:  79100, epoch: 203, train loss: 0.292124, valid loss: 0.297132, valid accuracy: 92.25%\n",
      "iteration:  79200, epoch: 203, train loss: 0.320075, valid loss: 0.297132, valid accuracy: 92.25%\n",
      "iteration:  79300, epoch: 203, train loss: 0.311115, valid loss: 0.297132, valid accuracy: 92.25%\n",
      "iteration:  79400, epoch: 204, train loss: 0.298753, valid loss: 0.310069, valid accuracy: 91.95%\n",
      "iteration:  79500, epoch: 204, train loss: 0.293715, valid loss: 0.310069, valid accuracy: 91.95%\n",
      "iteration:  79600, epoch: 204, train loss: 0.319647, valid loss: 0.310069, valid accuracy: 91.95%\n",
      "iteration:  79700, epoch: 204, train loss: 0.322400, valid loss: 0.310069, valid accuracy: 91.95%\n",
      "iteration:  79800, epoch: 205, train loss: 0.315309, valid loss: 0.312702, valid accuracy: 91.90%\n",
      "iteration:  79900, epoch: 205, train loss: 0.307679, valid loss: 0.312702, valid accuracy: 91.90%\n",
      "iteration:  80000, epoch: 205, train loss: 0.299798, valid loss: 0.312702, valid accuracy: 91.90%\n",
      "iteration:  80100, epoch: 205, train loss: 0.293733, valid loss: 0.312702, valid accuracy: 91.90%\n",
      "iteration:  80200, epoch: 206, train loss: 0.309355, valid loss: 0.331977, valid accuracy: 91.51%\n",
      "iteration:  80300, epoch: 206, train loss: 0.323652, valid loss: 0.331977, valid accuracy: 91.51%\n",
      "iteration:  80400, epoch: 206, train loss: 0.335406, valid loss: 0.331977, valid accuracy: 91.51%\n",
      "iteration:  80500, epoch: 206, train loss: 0.297983, valid loss: 0.331977, valid accuracy: 91.51%\n",
      "iteration:  80600, epoch: 207, train loss: 0.327393, valid loss: 0.297743, valid accuracy: 92.30%\n",
      "iteration:  80700, epoch: 207, train loss: 0.319277, valid loss: 0.297743, valid accuracy: 92.30%\n",
      "iteration:  80800, epoch: 207, train loss: 0.324964, valid loss: 0.297743, valid accuracy: 92.30%\n",
      "iteration:  80900, epoch: 207, train loss: 0.308339, valid loss: 0.297743, valid accuracy: 92.30%\n",
      "iteration:  81000, epoch: 208, train loss: 0.305227, valid loss: 0.292120, valid accuracy: 92.42%\n",
      "iteration:  81100, epoch: 208, train loss: 0.293606, valid loss: 0.292120, valid accuracy: 92.42%\n",
      "iteration:  81200, epoch: 208, train loss: 0.326918, valid loss: 0.292120, valid accuracy: 92.42%\n",
      "iteration:  81300, epoch: 208, train loss: 0.312290, valid loss: 0.292120, valid accuracy: 92.42%\n",
      "iteration:  81400, epoch: 209, train loss: 0.313309, valid loss: 0.346784, valid accuracy: 91.12%\n",
      "iteration:  81500, epoch: 209, train loss: 0.304133, valid loss: 0.346784, valid accuracy: 91.12%\n",
      "iteration:  81600, epoch: 209, train loss: 0.312676, valid loss: 0.346784, valid accuracy: 91.12%\n",
      "iteration:  81700, epoch: 209, train loss: 0.312181, valid loss: 0.346784, valid accuracy: 91.12%\n",
      "iteration:  81800, epoch: 210, train loss: 0.305800, valid loss: 0.306757, valid accuracy: 92.11%\n",
      "iteration:  81900, epoch: 210, train loss: 0.327369, valid loss: 0.306757, valid accuracy: 92.11%\n",
      "iteration:  82000, epoch: 210, train loss: 0.332525, valid loss: 0.306757, valid accuracy: 92.11%\n",
      "iteration:  82100, epoch: 210, train loss: 0.306683, valid loss: 0.306757, valid accuracy: 92.11%\n",
      "iteration:  82200, epoch: 211, train loss: 0.306486, valid loss: 0.345012, valid accuracy: 91.04%\n",
      "iteration:  82300, epoch: 211, train loss: 0.293238, valid loss: 0.345012, valid accuracy: 91.04%\n",
      "iteration:  82400, epoch: 211, train loss: 0.288403, valid loss: 0.345012, valid accuracy: 91.04%\n",
      "iteration:  82500, epoch: 211, train loss: 0.306975, valid loss: 0.345012, valid accuracy: 91.04%\n",
      "iteration:  82600, epoch: 212, train loss: 0.300013, valid loss: 0.304989, valid accuracy: 92.20%\n",
      "iteration:  82700, epoch: 212, train loss: 0.295668, valid loss: 0.304989, valid accuracy: 92.20%\n",
      "iteration:  82800, epoch: 212, train loss: 0.302649, valid loss: 0.304989, valid accuracy: 92.20%\n",
      "iteration:  82900, epoch: 213, train loss: 0.311102, valid loss: 0.297946, valid accuracy: 92.22%\n",
      "iteration:  83000, epoch: 213, train loss: 0.289513, valid loss: 0.297946, valid accuracy: 92.22%\n",
      "iteration:  83100, epoch: 213, train loss: 0.307562, valid loss: 0.297946, valid accuracy: 92.22%\n",
      "iteration:  83200, epoch: 213, train loss: 0.313777, valid loss: 0.297946, valid accuracy: 92.22%\n",
      "iteration:  83300, epoch: 214, train loss: 0.302959, valid loss: 0.302236, valid accuracy: 92.11%\n",
      "iteration:  83400, epoch: 214, train loss: 0.305284, valid loss: 0.302236, valid accuracy: 92.11%\n",
      "iteration:  83500, epoch: 214, train loss: 0.317057, valid loss: 0.302236, valid accuracy: 92.11%\n",
      "iteration:  83600, epoch: 214, train loss: 0.303116, valid loss: 0.302236, valid accuracy: 92.11%\n",
      "iteration:  83700, epoch: 215, train loss: 0.292373, valid loss: 0.276426, valid accuracy: 93.05%\n",
      "iteration:  83800, epoch: 215, train loss: 0.296381, valid loss: 0.276426, valid accuracy: 93.05%\n",
      "iteration:  83900, epoch: 215, train loss: 0.309578, valid loss: 0.276426, valid accuracy: 93.05%\n",
      "iteration:  84000, epoch: 215, train loss: 0.356631, valid loss: 0.276426, valid accuracy: 93.05%\n",
      "iteration:  84100, epoch: 216, train loss: 0.294994, valid loss: 0.286704, valid accuracy: 92.61%\n",
      "iteration:  84200, epoch: 216, train loss: 0.309899, valid loss: 0.286704, valid accuracy: 92.61%\n",
      "iteration:  84300, epoch: 216, train loss: 0.298449, valid loss: 0.286704, valid accuracy: 92.61%\n",
      "iteration:  84400, epoch: 216, train loss: 0.316566, valid loss: 0.286704, valid accuracy: 92.61%\n",
      "iteration:  84500, epoch: 217, train loss: 0.321862, valid loss: 0.297138, valid accuracy: 92.44%\n",
      "iteration:  84600, epoch: 217, train loss: 0.298732, valid loss: 0.297138, valid accuracy: 92.44%\n",
      "iteration:  84700, epoch: 217, train loss: 0.317446, valid loss: 0.297138, valid accuracy: 92.44%\n",
      "iteration:  84800, epoch: 217, train loss: 0.293932, valid loss: 0.297138, valid accuracy: 92.44%\n",
      "iteration:  84900, epoch: 218, train loss: 0.317671, valid loss: 0.310475, valid accuracy: 92.21%\n",
      "iteration:  85000, epoch: 218, train loss: 0.300799, valid loss: 0.310475, valid accuracy: 92.21%\n",
      "iteration:  85100, epoch: 218, train loss: 0.338917, valid loss: 0.310475, valid accuracy: 92.21%\n",
      "iteration:  85200, epoch: 218, train loss: 0.308010, valid loss: 0.310475, valid accuracy: 92.21%\n",
      "iteration:  85300, epoch: 219, train loss: 0.350824, valid loss: 0.291830, valid accuracy: 92.90%\n",
      "iteration:  85400, epoch: 219, train loss: 0.337622, valid loss: 0.291830, valid accuracy: 92.90%\n",
      "iteration:  85500, epoch: 219, train loss: 0.313915, valid loss: 0.291830, valid accuracy: 92.90%\n",
      "iteration:  85600, epoch: 219, train loss: 0.311700, valid loss: 0.291830, valid accuracy: 92.90%\n",
      "iteration:  85700, epoch: 220, train loss: 0.294972, valid loss: 0.285353, valid accuracy: 92.99%\n",
      "iteration:  85800, epoch: 220, train loss: 0.299706, valid loss: 0.285353, valid accuracy: 92.99%\n",
      "iteration:  85900, epoch: 220, train loss: 0.294932, valid loss: 0.285353, valid accuracy: 92.99%\n",
      "iteration:  86000, epoch: 220, train loss: 0.297352, valid loss: 0.285353, valid accuracy: 92.99%\n",
      "iteration:  86100, epoch: 221, train loss: 0.298271, valid loss: 0.272083, valid accuracy: 93.16%\n",
      "iteration:  86200, epoch: 221, train loss: 0.332104, valid loss: 0.272083, valid accuracy: 93.16%\n",
      "iteration:  86300, epoch: 221, train loss: 0.290209, valid loss: 0.272083, valid accuracy: 93.16%\n",
      "iteration:  86400, epoch: 221, train loss: 0.345007, valid loss: 0.272083, valid accuracy: 93.16%\n",
      "iteration:  86500, epoch: 222, train loss: 0.329925, valid loss: 0.286871, valid accuracy: 92.99%\n",
      "iteration:  86600, epoch: 222, train loss: 0.298328, valid loss: 0.286871, valid accuracy: 92.99%\n",
      "iteration:  86700, epoch: 222, train loss: 0.305236, valid loss: 0.286871, valid accuracy: 92.99%\n",
      "iteration:  86800, epoch: 222, train loss: 0.317382, valid loss: 0.286871, valid accuracy: 92.99%\n",
      "iteration:  86900, epoch: 223, train loss: 0.289032, valid loss: 0.290847, valid accuracy: 92.53%\n",
      "iteration:  87000, epoch: 223, train loss: 0.296709, valid loss: 0.290847, valid accuracy: 92.53%\n",
      "iteration:  87100, epoch: 223, train loss: 0.307297, valid loss: 0.290847, valid accuracy: 92.53%\n",
      "iteration:  87200, epoch: 224, train loss: 0.307278, valid loss: 0.330582, valid accuracy: 91.90%\n",
      "iteration:  87300, epoch: 224, train loss: 0.292614, valid loss: 0.330582, valid accuracy: 91.90%\n",
      "iteration:  87400, epoch: 224, train loss: 0.288176, valid loss: 0.330582, valid accuracy: 91.90%\n",
      "iteration:  87500, epoch: 224, train loss: 0.346156, valid loss: 0.330582, valid accuracy: 91.90%\n",
      "iteration:  87600, epoch: 225, train loss: 0.295223, valid loss: 0.302324, valid accuracy: 92.49%\n",
      "iteration:  87700, epoch: 225, train loss: 0.301240, valid loss: 0.302324, valid accuracy: 92.49%\n",
      "iteration:  87800, epoch: 225, train loss: 0.289333, valid loss: 0.302324, valid accuracy: 92.49%\n",
      "iteration:  87900, epoch: 225, train loss: 0.312019, valid loss: 0.302324, valid accuracy: 92.49%\n",
      "iteration:  88000, epoch: 226, train loss: 0.312075, valid loss: 0.300141, valid accuracy: 92.41%\n",
      "iteration:  88100, epoch: 226, train loss: 0.306139, valid loss: 0.300141, valid accuracy: 92.41%\n",
      "iteration:  88200, epoch: 226, train loss: 0.289523, valid loss: 0.300141, valid accuracy: 92.41%\n",
      "iteration:  88300, epoch: 226, train loss: 0.288723, valid loss: 0.300141, valid accuracy: 92.41%\n",
      "iteration:  88400, epoch: 227, train loss: 0.336486, valid loss: 0.284946, valid accuracy: 92.81%\n",
      "iteration:  88500, epoch: 227, train loss: 0.292018, valid loss: 0.284946, valid accuracy: 92.81%\n",
      "iteration:  88600, epoch: 227, train loss: 0.294631, valid loss: 0.284946, valid accuracy: 92.81%\n",
      "iteration:  88700, epoch: 227, train loss: 0.290037, valid loss: 0.284946, valid accuracy: 92.81%\n",
      "iteration:  88800, epoch: 228, train loss: 0.297780, valid loss: 0.305859, valid accuracy: 92.14%\n",
      "iteration:  88900, epoch: 228, train loss: 0.287087, valid loss: 0.305859, valid accuracy: 92.14%\n",
      "iteration:  89000, epoch: 228, train loss: 0.301410, valid loss: 0.305859, valid accuracy: 92.14%\n",
      "iteration:  89100, epoch: 228, train loss: 0.312190, valid loss: 0.305859, valid accuracy: 92.14%\n",
      "iteration:  89200, epoch: 229, train loss: 0.294785, valid loss: 0.286645, valid accuracy: 92.86%\n",
      "iteration:  89300, epoch: 229, train loss: 0.289497, valid loss: 0.286645, valid accuracy: 92.86%\n",
      "iteration:  89400, epoch: 229, train loss: 0.338107, valid loss: 0.286645, valid accuracy: 92.86%\n",
      "iteration:  89500, epoch: 229, train loss: 0.321463, valid loss: 0.286645, valid accuracy: 92.86%\n",
      "iteration:  89600, epoch: 230, train loss: 0.361264, valid loss: 0.296911, valid accuracy: 92.63%\n",
      "iteration:  89700, epoch: 230, train loss: 0.293575, valid loss: 0.296911, valid accuracy: 92.63%\n",
      "iteration:  89800, epoch: 230, train loss: 0.294378, valid loss: 0.296911, valid accuracy: 92.63%\n",
      "iteration:  89900, epoch: 230, train loss: 0.300313, valid loss: 0.296911, valid accuracy: 92.63%\n",
      "iteration:  90000, epoch: 231, train loss: 0.308157, valid loss: 0.282822, valid accuracy: 93.01%\n",
      "iteration:  90100, epoch: 231, train loss: 0.320018, valid loss: 0.282822, valid accuracy: 93.01%\n",
      "iteration:  90200, epoch: 231, train loss: 0.313198, valid loss: 0.282822, valid accuracy: 93.01%\n",
      "iteration:  90300, epoch: 231, train loss: 0.287658, valid loss: 0.282822, valid accuracy: 93.01%\n",
      "iteration:  90400, epoch: 232, train loss: 0.301609, valid loss: 0.293512, valid accuracy: 92.71%\n",
      "iteration:  90500, epoch: 232, train loss: 0.289818, valid loss: 0.293512, valid accuracy: 92.71%\n",
      "iteration:  90600, epoch: 232, train loss: 0.286877, valid loss: 0.293512, valid accuracy: 92.71%\n",
      "iteration:  90700, epoch: 232, train loss: 0.288410, valid loss: 0.293512, valid accuracy: 92.71%\n",
      "iteration:  90800, epoch: 233, train loss: 0.291670, valid loss: 0.292032, valid accuracy: 92.84%\n",
      "iteration:  90900, epoch: 233, train loss: 0.291248, valid loss: 0.292032, valid accuracy: 92.84%\n",
      "iteration:  91000, epoch: 233, train loss: 0.335411, valid loss: 0.292032, valid accuracy: 92.84%\n",
      "iteration:  91100, epoch: 233, train loss: 0.306005, valid loss: 0.292032, valid accuracy: 92.84%\n",
      "iteration:  91200, epoch: 234, train loss: 0.287496, valid loss: 0.286841, valid accuracy: 92.82%\n",
      "iteration:  91300, epoch: 234, train loss: 0.291181, valid loss: 0.286841, valid accuracy: 92.82%\n",
      "iteration:  91400, epoch: 234, train loss: 0.300307, valid loss: 0.286841, valid accuracy: 92.82%\n",
      "iteration:  91500, epoch: 235, train loss: 0.297408, valid loss: 0.287852, valid accuracy: 92.88%\n",
      "iteration:  91600, epoch: 235, train loss: 0.291471, valid loss: 0.287852, valid accuracy: 92.88%\n",
      "iteration:  91700, epoch: 235, train loss: 0.288783, valid loss: 0.287852, valid accuracy: 92.88%\n",
      "iteration:  91800, epoch: 235, train loss: 0.292672, valid loss: 0.287852, valid accuracy: 92.88%\n",
      "iteration:  91900, epoch: 236, train loss: 0.313118, valid loss: 0.284657, valid accuracy: 93.03%\n",
      "iteration:  92000, epoch: 236, train loss: 0.296133, valid loss: 0.284657, valid accuracy: 93.03%\n",
      "iteration:  92100, epoch: 236, train loss: 0.302044, valid loss: 0.284657, valid accuracy: 93.03%\n",
      "iteration:  92200, epoch: 236, train loss: 0.291474, valid loss: 0.284657, valid accuracy: 93.03%\n",
      "iteration:  92300, epoch: 237, train loss: 0.286157, valid loss: 0.279084, valid accuracy: 93.09%\n",
      "iteration:  92400, epoch: 237, train loss: 0.286781, valid loss: 0.279084, valid accuracy: 93.09%\n",
      "iteration:  92500, epoch: 237, train loss: 0.289477, valid loss: 0.279084, valid accuracy: 93.09%\n",
      "iteration:  92600, epoch: 237, train loss: 0.290606, valid loss: 0.279084, valid accuracy: 93.09%\n",
      "iteration:  92700, epoch: 238, train loss: 0.297865, valid loss: 0.278805, valid accuracy: 93.24%\n",
      "iteration:  92800, epoch: 238, train loss: 0.318676, valid loss: 0.278805, valid accuracy: 93.24%\n",
      "iteration:  92900, epoch: 238, train loss: 0.292076, valid loss: 0.278805, valid accuracy: 93.24%\n",
      "iteration:  93000, epoch: 238, train loss: 0.287352, valid loss: 0.278805, valid accuracy: 93.24%\n",
      "iteration:  93100, epoch: 239, train loss: 0.294217, valid loss: 0.279757, valid accuracy: 93.09%\n",
      "iteration:  93200, epoch: 239, train loss: 0.287659, valid loss: 0.279757, valid accuracy: 93.09%\n",
      "iteration:  93300, epoch: 239, train loss: 0.298567, valid loss: 0.279757, valid accuracy: 93.09%\n",
      "iteration:  93400, epoch: 239, train loss: 0.286618, valid loss: 0.279757, valid accuracy: 93.09%\n",
      "iteration:  93500, epoch: 240, train loss: 0.309114, valid loss: 0.283145, valid accuracy: 93.19%\n",
      "iteration:  93600, epoch: 240, train loss: 0.293748, valid loss: 0.283145, valid accuracy: 93.19%\n",
      "iteration:  93700, epoch: 240, train loss: 0.287746, valid loss: 0.283145, valid accuracy: 93.19%\n",
      "iteration:  93800, epoch: 240, train loss: 0.285613, valid loss: 0.283145, valid accuracy: 93.19%\n",
      "iteration:  93900, epoch: 241, train loss: 0.296170, valid loss: 0.286813, valid accuracy: 92.96%\n",
      "iteration:  94000, epoch: 241, train loss: 0.318082, valid loss: 0.286813, valid accuracy: 92.96%\n",
      "iteration:  94100, epoch: 241, train loss: 0.288155, valid loss: 0.286813, valid accuracy: 92.96%\n",
      "iteration:  94200, epoch: 241, train loss: 0.298832, valid loss: 0.286813, valid accuracy: 92.96%\n",
      "iteration:  94300, epoch: 242, train loss: 0.289523, valid loss: 0.290164, valid accuracy: 93.03%\n",
      "iteration:  94400, epoch: 242, train loss: 0.287750, valid loss: 0.290164, valid accuracy: 93.03%\n",
      "iteration:  94500, epoch: 242, train loss: 0.321508, valid loss: 0.290164, valid accuracy: 93.03%\n",
      "iteration:  94600, epoch: 242, train loss: 0.286816, valid loss: 0.290164, valid accuracy: 93.03%\n",
      "iteration:  94700, epoch: 243, train loss: 0.289302, valid loss: 0.284421, valid accuracy: 93.14%\n",
      "iteration:  94800, epoch: 243, train loss: 0.295373, valid loss: 0.284421, valid accuracy: 93.14%\n",
      "iteration:  94900, epoch: 243, train loss: 0.294041, valid loss: 0.284421, valid accuracy: 93.14%\n",
      "iteration:  95000, epoch: 243, train loss: 0.307444, valid loss: 0.284421, valid accuracy: 93.14%\n",
      "iteration:  95100, epoch: 244, train loss: 0.291653, valid loss: 0.278403, valid accuracy: 93.16%\n",
      "iteration:  95200, epoch: 244, train loss: 0.311594, valid loss: 0.278403, valid accuracy: 93.16%\n",
      "iteration:  95300, epoch: 244, train loss: 0.296037, valid loss: 0.278403, valid accuracy: 93.16%\n",
      "iteration:  95400, epoch: 244, train loss: 0.296585, valid loss: 0.278403, valid accuracy: 93.16%\n",
      "iteration:  95500, epoch: 245, train loss: 0.288600, valid loss: 0.277381, valid accuracy: 93.13%\n",
      "iteration:  95600, epoch: 245, train loss: 0.301220, valid loss: 0.277381, valid accuracy: 93.13%\n",
      "iteration:  95700, epoch: 245, train loss: 0.296987, valid loss: 0.277381, valid accuracy: 93.13%\n",
      "iteration:  95800, epoch: 246, train loss: 0.308425, valid loss: 0.283714, valid accuracy: 92.89%\n",
      "iteration:  95900, epoch: 246, train loss: 0.288393, valid loss: 0.283714, valid accuracy: 92.89%\n",
      "iteration:  96000, epoch: 246, train loss: 0.285475, valid loss: 0.283714, valid accuracy: 92.89%\n",
      "iteration:  96100, epoch: 246, train loss: 0.285401, valid loss: 0.283714, valid accuracy: 92.89%\n",
      "iteration:  96200, epoch: 247, train loss: 0.299769, valid loss: 0.288530, valid accuracy: 93.04%\n",
      "iteration:  96300, epoch: 247, train loss: 0.286367, valid loss: 0.288530, valid accuracy: 93.04%\n",
      "iteration:  96400, epoch: 247, train loss: 0.292775, valid loss: 0.288530, valid accuracy: 93.04%\n",
      "iteration:  96500, epoch: 247, train loss: 0.299638, valid loss: 0.288530, valid accuracy: 93.04%\n",
      "iteration:  96600, epoch: 248, train loss: 0.296127, valid loss: 0.279347, valid accuracy: 93.23%\n",
      "iteration:  96700, epoch: 248, train loss: 0.286699, valid loss: 0.279347, valid accuracy: 93.23%\n",
      "iteration:  96800, epoch: 248, train loss: 0.288097, valid loss: 0.279347, valid accuracy: 93.23%\n",
      "iteration:  96900, epoch: 248, train loss: 0.285106, valid loss: 0.279347, valid accuracy: 93.23%\n",
      "iteration:  97000, epoch: 249, train loss: 0.286976, valid loss: 0.281666, valid accuracy: 93.17%\n",
      "iteration:  97100, epoch: 249, train loss: 0.324073, valid loss: 0.281666, valid accuracy: 93.17%\n",
      "iteration:  97200, epoch: 249, train loss: 0.292129, valid loss: 0.281666, valid accuracy: 93.17%\n",
      "iteration:  97300, epoch: 249, train loss: 0.290673, valid loss: 0.281666, valid accuracy: 93.17%\n",
      "iteration:  97400, epoch: 250, train loss: 0.288765, valid loss: 0.280242, valid accuracy: 93.28%\n",
      "iteration:  97500, epoch: 250, train loss: 0.289531, valid loss: 0.280242, valid accuracy: 93.28%\n",
      "iteration:  97600, epoch: 250, train loss: 0.286935, valid loss: 0.280242, valid accuracy: 93.28%\n",
      "iteration:  97700, epoch: 250, train loss: 0.286880, valid loss: 0.280242, valid accuracy: 93.28%\n",
      "iteration:  97800, epoch: 251, train loss: 0.289798, valid loss: 0.275489, valid accuracy: 93.35%\n",
      "iteration:  97900, epoch: 251, train loss: 0.301143, valid loss: 0.275489, valid accuracy: 93.35%\n",
      "iteration:  98000, epoch: 251, train loss: 0.287216, valid loss: 0.275489, valid accuracy: 93.35%\n"
     ]
    }
   ],
   "source": [
    "import paddle\n",
    "import paddle.fluid as fluid\n",
    "from paddle.utils.plot import Ploter\n",
    "import numpy as np\n",
    "import time\n",
    "import math\n",
    "import os\n",
    "\n",
    "epoch_num = 300   # 训练周期，取值一般为[1,300]\n",
    "train_batch = 128 # 训练批次，取值一般为[1,256]\n",
    "valid_batch = 128 # 验证批次，取值一般为[1,256]\n",
    "displays = 100    # 显示迭代\n",
    "\n",
    "start_lr = 0.00001                         # 开始学习率，取值一般为[1e-8,5e-1]\n",
    "based_lr = 0.1                             # 基础学习率，取值一般为[1e-8,5e-1]\n",
    "epoch_iters = math.ceil(50000/train_batch) # 每轮迭代数\n",
    "warmup_iter = 10 * epoch_iters             # 预热迭代数，取值一般为[1,10]\n",
    "\n",
    "momentum = 0.9     # 优化器动量\n",
    "l2_decay = 0.00005 # 正则化系数，取值一般为[1e-5,5e-4]\n",
    "epsilon = 0.05     # 标签平滑率，取值一般为[1e-2,1e-1]\n",
    "\n",
    "checkpoint = False                   # 断点标识\n",
    "model_path = './work/out/ssrnet'     # 模型路径\n",
    "result_txt = './work/out/result.txt' # 结果文件\n",
    "class_num  = 10                      # 类别数量\n",
    "\n",
    "with fluid.dygraph.guard():\n",
    "    # 准备数据\n",
    "    train_reader = paddle.batch(\n",
    "        reader=paddle.reader.shuffle(reader=paddle.dataset.cifar.train10(), buf_size=50000),\n",
    "        batch_size=train_batch)\n",
    "    \n",
    "    valid_reader = paddle.batch(\n",
    "        reader=paddle.dataset.cifar.test10(),\n",
    "        batch_size=valid_batch)\n",
    "    \n",
    "    # 声明模型\n",
    "    model = ResNet()\n",
    "    \n",
    "    # 优化算法\n",
    "    consine_lr = fluid.layers.cosine_decay(based_lr, epoch_iters, epoch_num) # 余弦衰减策略\n",
    "    decayed_lr = fluid.layers.linear_lr_warmup(consine_lr, warmup_iter, start_lr, based_lr) # 线性预热策略\n",
    "    \n",
    "    optimizer = fluid.optimizer.Momentum(\n",
    "        learning_rate=decayed_lr,                           # 衰减学习策略\n",
    "        momentum=momentum,                                  # 优化动量系数\n",
    "        regularization=fluid.regularizer.L2Decay(l2_decay), # 正则衰减系数\n",
    "        parameter_list=model.parameters())\n",
    "    \n",
    "    # 加载断点\n",
    "    if checkpoint: # 是否加载断点文件\n",
    "        model_dict, optimizer_dict = fluid.load_dygraph(model_path) # 加载断点参数\n",
    "        model.set_dict(model_dict)                                  # 设置权重参数\n",
    "        optimizer.set_dict(optimizer_dict)                          # 设置优化参数\n",
    "    else:          # 否则删除结果文件\n",
    "        if os.path.exists(result_txt): # 如果存在结果文件\n",
    "            os.remove(result_txt)      # 那么删除结果文件\n",
    "    \n",
    "    # 初始训练\n",
    "    avg_train_loss = 0 # 平均训练损失\n",
    "    avg_valid_loss = 0 # 平均验证损失\n",
    "    avg_valid_accu = 0 # 平均验证精度\n",
    "    \n",
    "    iterator = 1                                # 迭代次数\n",
    "    train_prompt = \"Train loss\"                 # 训练标签\n",
    "    valid_prompt = \"Valid loss\"                 # 验证标签\n",
    "    ploter = Ploter(train_prompt, valid_prompt) # 训练图像\n",
    "    \n",
    "    best_epoch = 0           # 最好周期\n",
    "    best_accu = 0            # 最好精度\n",
    "    best_loss = 100.0        # 最好损失\n",
    "    train_time = time.time() # 训练时间\n",
    "    \n",
    "    # 开始训练\n",
    "    for epoch_id in range(epoch_num):\n",
    "        # 训练模型\n",
    "        model.train() # 设置训练\n",
    "        for batch_id, train_data in enumerate(train_reader()):\n",
    "            # 读取数据\n",
    "            image_data = np.array([x[0] for x in train_data]).reshape((-1, 3, 32, 32)).astype(np.float32) # 读取图像数据\n",
    "            image_data = train_augment(image_data)                                                        # 使用数据增强\n",
    "            image = fluid.dygraph.to_variable(image_data)                                                 # 转换数据类型\n",
    "\n",
    "            label_data = np.array([x[1] for x in train_data]).astype(np.int64)                        # 读取标签数据\n",
    "            label = fluid.dygraph.to_variable(label_data)                                             # 转换数据类型\n",
    "            label = fluid.layers.label_smooth(label=fluid.one_hot(label, class_num), epsilon=epsilon) # 使用标签平滑\n",
    "            label.stop_gradient = True                                                                # 停止梯度传播\n",
    "\n",
    "            # 前向传播\n",
    "            infer = model(image)\n",
    "            \n",
    "            # 计算损失\n",
    "            loss = fluid.layers.cross_entropy(infer, label, soft_label=True)\n",
    "            train_loss = fluid.layers.mean(loss)\n",
    "            \n",
    "            # 反向传播\n",
    "            train_loss.backward()\n",
    "            optimizer.minimize(train_loss)\n",
    "            model.clear_gradients()\n",
    "            \n",
    "            # 显示结果\n",
    "            if iterator % displays == 0:\n",
    "                # 显示图像\n",
    "                avg_train_loss = train_loss.numpy()[0]                # 设置训练损失\n",
    "                ploter.append(train_prompt, iterator, avg_train_loss) # 添加训练图像\n",
    "                ploter.plot()                                         # 显示训练图像\n",
    "                \n",
    "                # 打印结果\n",
    "                print(\"iteration: {:6d}, epoch: {:3d}, train loss: {:.6f}, valid loss: {:.6f}, valid accuracy: {:.2%}\".format(\n",
    "                    iterator, epoch_id+1, avg_train_loss, avg_valid_loss, avg_valid_accu))\n",
    "                \n",
    "                # 写入文件\n",
    "                with open(result_txt, 'a') as file:\n",
    "                    file.write(\"iteration: {:6d}, epoch: {:3d}, train loss: {:.6f}, valid loss: {:.6f}, valid accuracy: {:.2%}\\n\".format(\n",
    "                        iterator, epoch_id+1, avg_train_loss, avg_valid_loss, avg_valid_accu))\n",
    "            \n",
    "            # 增加迭代\n",
    "            iterator += 1\n",
    "            \n",
    "        # 验证模型\n",
    "        valid_loss_list = [] # 验证损失列表\n",
    "        valid_accu_list = [] # 验证精度列表\n",
    "        \n",
    "        model.eval()   # 设置验证\n",
    "        for batch_id, valid_data in enumerate(valid_reader()):\n",
    "            # 读取数据\n",
    "            image_data = np.array([x[0] for x in valid_data]).reshape((-1, 3, 32, 32)).astype(np.float32) # 读取图像数据\n",
    "            image_data = valid_augment(image_data)                                                        # 使用图像增强\n",
    "            image = fluid.dygraph.to_variable(image_data)                                                 # 转换数据类型\n",
    "            \n",
    "            label_data = np.array([x[1] for x in valid_data]).reshape((-1, 1)).astype(np.int64) # 读取标签数据\n",
    "            label = fluid.dygraph.to_variable(label_data)                                       # 转换数据类型\n",
    "            label.stop_gradient = True                                                          # 停止梯度传播\n",
    "            \n",
    "            # 前向传播\n",
    "            infer = model(image)\n",
    "            \n",
    "            # 计算精度\n",
    "            valid_accu = fluid.layers.accuracy(infer,label)\n",
    "            \n",
    "            valid_accu_list.append(valid_accu.numpy())\n",
    "            \n",
    "            # 计算损失\n",
    "            loss = fluid.layers.cross_entropy(infer, label)\n",
    "            valid_loss = fluid.layers.mean(loss)\n",
    "            \n",
    "            valid_loss_list.append(valid_loss.numpy())\n",
    "        \n",
    "        # 设置结果\n",
    "        avg_valid_accu = np.mean(valid_accu_list)             # 设置验证精度\n",
    "        \n",
    "        avg_valid_loss = np.mean(valid_loss_list)             # 设置验证损失\n",
    "        ploter.append(valid_prompt, iterator, avg_valid_loss) # 添加训练图像\n",
    "        \n",
    "        # 保存模型\n",
    "        fluid.save_dygraph(model.state_dict(), model_path)     # 保存权重参数\n",
    "        fluid.save_dygraph(optimizer.state_dict(), model_path) # 保存优化参数\n",
    "        \n",
    "        if avg_valid_loss < best_loss:\n",
    "            fluid.save_dygraph(model.state_dict(), model_path + '-best') # 保存权重\n",
    "            \n",
    "            best_epoch = epoch_id + 1                                    # 更新迭代\n",
    "            best_accu = avg_valid_accu                                   # 更新精度\n",
    "            best_loss = avg_valid_loss                                   # 更新损失\n",
    "    \n",
    "    # 显示结果\n",
    "    train_time = time.time() - train_time # 设置训练时间\n",
    "    print('complete - train time: {:.0f}s, best epoch: {:3d}, best loss: {:.6f}, best accuracy: {:.2%}'.format(\n",
    "        train_time, best_epoch, best_loss, best_accu))\n",
    "    \n",
    "    # 写入文件\n",
    "    with open(result_txt, 'a') as file:\n",
    "        file.write('complete - train time: {:.0f}s, best epoch: {:3d}, best loss: {:.6f}, best accuracy: {:.2%}\\n'.format(\n",
    "            train_time, best_epoch, best_loss, best_accu))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "### 模型预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import paddle.fluid as fluid\n",
    "from PIL import Image\n",
    "import numpy as np\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "image_path = './work/out/img.png' # 图片路径\n",
    "model_path = './work/out/ssrnet-best' # 模型路径\n",
    "\n",
    "# 加载图像\n",
    "def load_image(image_path):\n",
    "    \"\"\"\n",
    "    功能:\n",
    "        读取图像并转换到输入格式\n",
    "    输入:\n",
    "        image_path - 输入图像路径\n",
    "    输出:\n",
    "        image - 输出图像\n",
    "    \"\"\"\n",
    "    # 读取图像\n",
    "    image = Image.open(image_path) # 打开图像文件\n",
    "    \n",
    "    # 转换格式\n",
    "    image = image.resize((32, 32), Image.ANTIALIAS) # 调整图像大小\n",
    "    image = np.array(image, dtype=np.float32) # 转换数据格式，数据类型转换为float32\n",
    "\n",
    "    # 减去均值\n",
    "    mean = np.array([0.4914, 0.4822, 0.4465]).reshape((1, 1, -1)) # cifar数据集通道平均值\n",
    "    stdv = np.array([0.2471, 0.2435, 0.2616]).reshape((1, 1, -1)) # cifar数据集通道标准差\n",
    "    \n",
    "    image = (image/255.0 - mean) / stdv # 对图像进行归一化\n",
    "    image = image.transpose((2, 0, 1)).astype(np.float32) # 数据格式从HWC转换为CHW，数据类型转换为float32\n",
    "    \n",
    "    # 增加维度\n",
    "    image = np.expand_dims(image, axis=0) # 增加数据维度\n",
    "    \n",
    "    return image\n",
    "\n",
    "# 预测图像\n",
    "with fluid.dygraph.guard():\n",
    "    # 读取图像\n",
    "    image = load_image(image_path)\n",
    "    image = fluid.dygraph.to_variable(image)\n",
    "    \n",
    "    # 加载模型\n",
    "    model = ResNet()                               # 加载模型\n",
    "    model_dict, _ = fluid.load_dygraph(model_path) # 加载权重\n",
    "    model.set_dict(model_dict)                     # 设置权重\n",
    "    model.eval()                                   # 设置验证\n",
    "    \n",
    "    # 前向传播\n",
    "    infer_time = time.time()              # 推断开始时间\n",
    "    infer = model(image)\n",
    "    infer_time = time.time() - infer_time # 推断结束时间\n",
    "    \n",
    "    # 显示结果\n",
    "    vlist = [\"airplane\", \"automobile\", \"bird\", \"cat\", \"deer\", \"dog\", \"frog\", \"horse\", \"ship\", \"truck\"] # 标签名称列表\n",
    "    print('infer time: {:f}s, infer value: {}'.format(infer_time, vlist[np.argmax(infer.numpy())]) )\n",
    "    \n",
    "    image = Image.open(image_path) # 打开图像文件\n",
    "    plt.figure(figsize=(3, 3))     # 设置显示大小\n",
    "    plt.imshow(image)              # 设置显示图像\n",
    "    plt.show()                     # 显示图像文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PaddlePaddle 1.8.4 (Python 3.5)",
   "language": "python",
   "name": "py35-paddle1.2.0"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
